Notebook

Lesson3-Pipeline API

In [1]:
# Research environment functions imports
from quantopian.research import run_pipeline

# Pipeline imports
from quantopian.pipeline.data import EquityPricing # OHLCV data for Global equity
from quantopian.pipeline.data import factset # FactSet data set

from quantopian.pipeline.domain import IN_EQUITIES # Indian equities

from quantopian.pipeline import Pipeline

# Other libraries imports
import matplotlib.pyplot as plt
import pandas as pd
In [2]:
# Select time period to inspect
start_period = '2015-01-15'
end_period= '2016-01-15'


# Pipeline definition
pipe=Pipeline(
    columns={   'price':EquityPricing.close.latest,
                'volume':EquityPricing.volume.latest,
                'mkt_cap':factset.Fundamentals.mkt_val.latest,
                'economy':factset.RBICSFocus.l1_name.latest,
                'sector':factset.RBICSFocus.l2_name.latest,
                'subsector':factset.RBICSFocus.l3_name.latest,
                'listing exchange':factset.EquityMetadata.primary_fsym_security_id.latest,
                'listing currency':factset.EquityMetadata.listing_currency.latest,
                'security type':factset.EquityMetadata.security_type.latest
            },
            domain=IN_EQUITIES)
In [3]:
# Pipeline execution
result= run_pipeline(
                pipe,
                start_period,
                end_period)

result.loc['2016-01-15',:][result['security type']=='ETF_ETF']

/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:7: UserWarning: Boolean Series key will be reindexed to match DataFrame index.
  import sys
Out[3]:
economy listing currency listing exchange mkt_cap price sector security type subsector volume
2016-01-15 00:00:00+00:00 Equity(1180120417450061 [533891]) None INR K6Q51M-S NaN 2281.23 None ETF_ETF None 132.0
Equity(1180138149528400 [537007]) None INR DQWL1Q-S NaN 82.80 None ETF_ETF None 0.0
Equity(1180154845086786 [590098]) None INR SFKN75-S NaN 2408.74 None ETF_ETF None 317.0
Equity(1182212738861907 [532985]) None INR QVLBG8-S NaN 251.10 None ETF_ETF None 34129.0
Equity(1182221260184140 [590110]) None INR KNH9QH-S NaN 796.50 None ETF_ETF None 0.0
Equity(1182319472826708 [537708]) None INR CQG1H7-S NaN 763.25 None ETF_ETF None 253.0
Equity(1183423884905041 [590095]) None INR R3J7G1-S NaN 2364.65 None ETF_ETF None 855.0
Equity(1183449654708811 [539031]) None INR GHN7GJ-S NaN 77.00 None ETF_ETF None 1.0
Equity(1184497442509136 [590103]) None INR LN1DY6-S NaN 763.14 None ETF_ETF None 1541.0
Equity(1184544736168524 [539487]) None INR C4S2TD-S NaN 210.00 None ETF_ETF None 0.0
Equity(1186618685014600 [533230]) None INR JWV28C-S NaN 2420.00 None ETF_ETF None 69.0
Equity(1187688436356947 [533719]) None INR BZMTJD-S NaN 2401.00 None ETF_ETF None 0.0
Equity(1187757423604294 [533172]) None INR QKXV85-S NaN 2152.00 None ETF_ETF None 0.0
Equity(1187783242963271 [555555]) None INR SW6HKW-S NaN 275.00 None ETF_ETF None 0.0
Equity(1187834464065868 [533570]) None INR MJJ6MQ-S NaN 2275.00 None ETF_ETF None 5.0
Equity(1188865036862285 [539142]) None INR PVZF24-S NaN 6800.00 None ETF_ETF None 0.0
Equity(1199812910929736 [590099]) None INR VYR27H-S NaN 1188.95 None ETF_ETF None 14.0
Equity(1199929426130518 [590115]) None INR JSJ2QL-S NaN 73.20 None ETF_ETF None 4.0
Equity(1199937448003148 [539312]) None INR DSQ8PF-S NaN 252.00 None ETF_ETF None 625.0
Equity(1203189323092818 [590109]) None INR DVRGTB-S NaN 181.00 None ETF_ETF None 0.0
Equity(1203201941527126 [539517]) None INR PG25QJ-S NaN 2478.00 None ETF_ETF None 0.0
Equity(1204275682620231 [590104]) None INR HQ6LJM-S NaN 195.00 None ETF_ETF None 103.0
Equity(1209846522205262 [537008]) None INR B3G4W2-S NaN 83.05 None ETF_ETF None 0.0
Equity(1210911659996739 [533244]) None INR DSNXPR-S NaN 2422.00 None ETF_ETF None 4.0
Equity(1211971962950478 [533385]) None INR H2SZ30-S NaN 324.96 None ETF_ETF None 913.0
Equity(1211989510141270 [537483]) None INR V25NLK-S NaN 75.10 None ETF_ETF None 0.0
Equity(1214192947444049 [590096]) None INR M76L0Q-S NaN 1000.00 None ETF_ETF None 279886.0
Equity(1214197124257859 [538683]) None INR PTDS8G-S NaN 259.07 None ETF_ETF None 0.0
Equity(1214231014224211 [535276]) None INR FC5Y88-S NaN 259.00 None ETF_ETF None 12.0
Equity(1215167686201680 [538057]) None INR PB8JF6-S NaN 20.38 None ETF_ETF None 31327.0
Equity(1216301205897799 [539313]) None INR JWWF81-S NaN 766.50 None ETF_ETF None 0.0
Equity(1216301222481238 [590101]) None INR RPPDJK-S NaN 2335.03 None ETF_ETF None 237.0
Equity(1216408580281155 [590107]) None INR QVKXNW-S NaN 251.00 None ETF_ETF None 525.0
Equity(1216447302089043 [533408]) None INR BDGNFW-S NaN 2386.00 None ETF_ETF None 0.0
Equity(1217401302758993 [533888]) None INR J4SRY0-S NaN 1705.00 None ETF_ETF None 0.0
Equity(1217465173563981 [590113]) None INR MNV9CK-S NaN 1771.00 None ETF_ETF None 0.0
Equity(1218474760951633 [539516]) None INR NVX0N0-S NaN 856.00 None ETF_ETF None 0.0
Equity(1220759733548376 [590106]) None INR PKW7BS-S NaN 1590.00 None ETF_ETF None 216.0
Equity(1220828133280337 [537484]) None INR HSK9VH-S NaN 77.00 None ETF_ETF None 0.0
Equity(1221880819502158 [590097]) None INR PJSL56-S NaN 233.10 None ETF_ETF None 1483.0
Equity(1223967854966602 [590108]) None INR M500CH-S NaN 271.30 None ETF_ETF None 24.0
Equity(1225080572499792 [534631]) None INR J92BSF-S NaN 2455.00 None ETF_ETF None 0.0

How to get the name and symbol of a security

In [4]:
# The index level=1 contains the security or asset object
# It has several attributes. Lets see what they are by looking at the docstr
first_security = result.index.get_level_values(level=1)[0]
help(first_security)

Help on Equity object:

class Equity(Asset)
 |  Method resolution order:
 |      Equity
 |      Asset
 |      __builtin__.object
 |  
 |  Data descriptors defined here:
 |  
 |  security_end_date
 |      DEPRECATION: This property should be deprecated and is only present for
 |      backwards compatibility
 |  
 |  security_name
 |      DEPRECATION: This property should be deprecated and is only present for
 |      backwards compatibility
 |  
 |  security_start_date
 |      DEPRECATION: This property should be deprecated and is only present for
 |      backwards compatibility
 |  
 |  ----------------------------------------------------------------------
 |  Data and other attributes defined here:
 |  
 |  __new__ = <built-in method __new__ of type object>
 |      T.__new__(S, ...) -> a new object with type S, a subtype of T
 |  
 |  __pyx_vtable__ = <capsule object NULL>
 |  
 |  ----------------------------------------------------------------------
 |  Methods inherited from Asset:
 |  
 |  __eq__(...)
 |      x.__eq__(y) <==> x==y
 |  
 |  __ge__(...)
 |      x.__ge__(y) <==> x>=y
 |  
 |  __gt__(...)
 |      x.__gt__(y) <==> x>y
 |  
 |  __hash__(...)
 |      x.__hash__() <==> hash(x)
 |  
 |  __index__(...)
 |      x[y:z] <==> x[y.__index__():z.__index__()]
 |  
 |  __init__(...)
 |      x.__init__(...) initializes x; see help(type(x)) for signature
 |  
 |  __int__(...)
 |      x.__int__() <==> int(x)
 |  
 |  __le__(...)
 |      x.__le__(y) <==> x<=y
 |  
 |  __long__(...)
 |      x.__long__() <==> long(x)
 |  
 |  __lt__(...)
 |      x.__lt__(y) <==> x<y
 |  
 |  __ne__(...)
 |      x.__ne__(y) <==> x!=y
 |  
 |  __reduce__(...)
 |      Asset.__reduce__(self)
 |      
 |      Function used by pickle to determine how to serialize/deserialize this
 |      class.  Should return a tuple whose first element is self.__class__,
 |      and whose second element is a tuple of all the attributes that should
 |      be serialized/deserialized during pickling.
 |  
 |  __repr__(...)
 |      x.__repr__() <==> repr(x)
 |  
 |  from_dict(...)
 |      Asset.from_dict(type cls, dict_)
 |      
 |      Build an Asset instance from a dict.
 |  
 |  is_alive_for_session(...)
 |      Asset.is_alive_for_session(self, session_label)
 |      
 |      Returns whether the asset is alive at the given dt.
 |      
 |      Parameters
 |      ----------
 |      session_label: pd.Timestamp
 |          The desired session label to check. (midnight UTC)
 |      
 |      Returns
 |      -------
 |      boolean: whether the asset is alive at the given dt.
 |  
 |  is_exchange_open(...)
 |      Asset.is_exchange_open(self, dt_minute)
 |      
 |      Parameters
 |      ----------
 |      dt_minute: pd.Timestamp (UTC, tz-aware)
 |          The minute to check.
 |      
 |      Returns
 |      -------
 |      boolean: whether the asset's exchange is open at the given minute.
 |  
 |  to_dict(...)
 |      Asset.to_dict(self)
 |      
 |      Convert to a python dict.
 |  
 |  ----------------------------------------------------------------------
 |  Data descriptors inherited from Asset:
 |  
 |  asset_name
 |  
 |  auto_close_date
 |  
 |  country_code
 |  
 |  end_date
 |  
 |  exchange
 |  
 |  exchange_full
 |  
 |  exchange_info
 |  
 |  first_traded
 |      first_traded: object
 |  
 |  price_multiplier
 |  
 |  sid
 |  
 |  start_date
 |  
 |  symbol
 |  
 |  tick_size

Some interesting attributes are | asset_name | exchange | exchange_full | symbol | sid

In [5]:
# Create columns to expose these attributes
result['asset_name'] = [str(asset.asset_name) for asset in result.index.get_level_values(level=1)]
result['exchange'] = [str(asset.exchange) for asset in result.index.get_level_values(level=1)]
result['exchange_full'] = [str(asset.exchange_full) for asset in result.index.get_level_values(level=1)]
result['symbol'] = [str(asset.symbol) for asset in result.index.get_level_values(level=1)]
result['sid'] = [str(asset.sid) for asset in result.index.get_level_values(level=1)]

# Look at the new dataframe and see how the different exchanges handle symbols
result.groupby('exchange').first()
Out[5]:
economy listing currency listing exchange mkt_cap price sector security type subsector volume asset_name exchange_full symbol sid
exchange
BOM Healthcare INR R5YP6Q-S 5.104200e+08 140.0 Healthcare Services SHARE Patient Care 4585.0 Dr. Agarwal'S Eye Hospital Limited BSE Ltd. (BOM) 526783 1178883482794310
NSE Consumer Non-Cyclicals INR CL2QD7-S 1.126230e+09 220.9 Food and Tobacco Production SHARE Food and Beverage Production 111.0 United Nilgiri Tea Estates Co. Ltd. National Stock Exchange of India (NSE) UNITEDTEA 1178913799819604
In [6]:
# From above it can be seen that the BOM exchange uses numbers for tickers but the NSE uses alpha characters
# To reference a given asset one can use the standard 'symbols' method
# However, use the SID rather than the ticker. The ticker only works for US securities.
# Don't put the sid in quotes
argawals = symbols(1178883482794310)
argawals
Out[6]:
Equity(1178883482794310 [526783])
In [7]:
# check it's the right one by lookiing at the name
argawals.asset_name
Out[7]:
u"Dr. Agarwal'S Eye Hospital Limited"
In [8]:
# a little trick to remove the preceding 'u'...
# change the value to an ascii string rather than unicode
str(argawals.asset_name)
Out[8]:
"Dr. Agarwal'S Eye Hospital Limited"

So, referencing a global security is identical to a US security except use the SID rather than the ticker.

In [ ]: