Close = USEquityPricing.close.latest
    Open = USEquityPricing.open.latest
    pb_ratio = valuation_ratios.pb_ratio.latest
    pb_ratio_dec = pb_ratio.quantiles(10) #Adjusted close price / Book Value Per Share.
    forward_earning_yield = valuation_ratios.forward_earning_yield.latest
    forward_earning_yield_dec = forward_earning_yield.quantiles(10) #Estimated Earnings Per Share / Price
    Intraday_Return = (Close - Open)/Close
    sector = Sector ()
    
pipe = Pipeline(
    columns={
        'Close': USEquityPricing.close.latest,
        'Open' : Open,
        'Pb Ratio' : pb_ratio,
        'Pb Ratio Dec': pb_ratio_dec,
        'Forward Earning Yield' : forward_earning_yield,
        'Forward Earning Yield Dec' : forward_earning_yield_dec,
        'IntradayReturn': Intraday_Return,
        'Sector' : sector}, 
    screen=Q500US()
)

Q500 = run_pipeline(pipe, '2018-01-01', '2019-01-01')
Q500.head(10)

Q500_sectors = Q500.groupby(['Pb Ratio Dec', 'Forward Earning Yield Dec','Sector'])['IntradayReturn'].mean()

Q500_sectors.head(10)

heatmap = np.array(Q500_sectors[:])
data = np.resize(heatmap, 100).reshape(10,10)

sns.heatmap(data, linewidths=0, cmap='RdYlGn', xticklabels='Pb Ratio Dec', yticklabels='Forward Earning Yield Dec', robust=True, annot=True);

import pandas as pd
import numpy as np
from quantopian.pipeline import Pipeline, CustomFactor
from quantopian.pipeline.data import Fundamentals
from quantopian.pipeline.factors import Latest
from quantopian.pipeline.filters import Q500US
from quantopian.research import run_pipeline
from time import strftime
from datetime import timedelta, datetime
from quantopian.pipeline.data.builtin import USEquityPricing
import seaborn as sns
import alphalens
import matplotlib.pyplot as plt
import talib as ta
from quantopian.pipeline.classifiers.morningstar import Sector
from quantopian.pipeline.data.user_5bc7b3c7439bfa004bdb1112 import merged

MORNINGSTAR_SECTOR_CODES = {
     -1: 'Misc',
    101: 'Basic Materials',
    102: 'Consumer Cyclical',
    103: 'Financial Services',
    104: 'Real Estate',
    205: 'Consumer Defensive',
    206: 'Healthcare',
    207: 'Utilities',
    308: 'Communication Services',
    309: 'Energy',
    310: 'Industrials',
    311: 'Technology' ,    
}

def auto_heatmap(factor1, factor2, factor_1_name, factor_2_name, quantile_groups = 4, start_date = '2018-01-01', end_date = '2019-01-01'):
    class PreviousClose(CustomFactor):  
        inputs = [USEquityPricing.close]  
        window_length = 1

        def compute(self, today, assets, out, close):  
            out[:] = close[0]

    class PreviousOpen(CustomFactor):  
        inputs = [USEquityPricing.open]  
        window_length = 1

        def compute(self, today, assets, out, open):  
            out[:] = open[0]

    def make_pipeline():
        # define our fundamental factor pipeline
        pipe = Pipeline()
        
        universe = Q500US()
        
        factor_1 = Latest([factor1])
        factor_2 = Latest([factor2])

        factor_1_dec = factor_1.quantiles(quantile_groups, mask=universe)
        factor_2_dec = factor_2.quantiles(quantile_groups, mask=universe)


    #     PreviousClose(window_length = 1) is the same as USEquityPricing.close.latest
        previous_close_price = PreviousClose(window_length = 2)
        close_price = USEquityPricing.close.latest
        open_price = USEquityPricing.open.latest

        Overnight_return = open_price / previous_close_price - 1
        Intraday_return = close_price / open_price - 1

        pipe = Pipeline(
            columns = {
                'open':open_price,
                'close':close_price,
                'close_previous':previous_close_price,
                'Overnight_return':Overnight_return,
                'Intraday_return':Intraday_return,
                factor_1_name:factor_1,
                factor_2_name:factor_2,
                factor_1_name+" quantile":factor_1_dec,
                factor_2_name+" quantile":factor_2_dec,
                'Sector': Sector(mask=universe),
                'first_30': merged.first_30.latest,
                'last_30': merged.last_30.latest
            },
            screen=universe
        )

        return pipe

    pipe = make_pipeline()

    results = run_pipeline(pipe, start_date, end_date).dropna()
    
    decile_means = results.groupby([factor_1_name+" quantile", factor_2_name+" quantile"])['Intraday_return', 'Overnight_return', 'first_30', 'last_30'].mean()
    decile_count = results.groupby([factor_1_name+" quantile", factor_2_name+" quantile"])['Overnight_return'].count()
    
    for ret_type in ['Intraday_return', 'Overnight_return', 'first_30', 'last_30']:
        sns.heatmap(decile_means[ret_type].unstack(), annot=True, linewidths=.5, fmt=".2%", center=0, cmap=sns.diverging_palette(20, 150, as_cmap=True), vmin=-0.005, vmax=0.005,).set_title(ret_type)
        plt.figure()
    
    sns.heatmap(decile_count.unstack(), annot=True, linewidths=.5, fmt='g', cmap="Blues").set_title('Number of Observations')
    plt.figure()
    
    decile_means_sector = results.groupby([factor_1_name+" quantile", factor_2_name+" quantile", "Sector"])['Intraday_return', 'Overnight_return', 'first_30', 'last_30'].mean()
    
    
    for ret_type in ['Intraday_return', 'Overnight_return', 'first_30', 'last_30']:
        def facet_heatmap(data, color, value_col, **kws):
            data = data.pivot(index=factor_1_name+" quantile", columns=factor_2_name+" quantile", values = value_col)
            sns.heatmap(data, annot=True, linewidths=.5, fmt=".2%", center=0, cmap=sns.diverging_palette(20, 150, as_cmap=True), vmin=-0.005, vmax=0.005, cbar = False, **kws)

        with sns.plotting_context(font_scale=5):
            g = sns.FacetGrid(decile_means_sector.reset_index(), col="Sector", col_wrap=3, size=4.5, aspect=1)

        g = g.map_dataframe(facet_heatmap, value_col = ret_type)

        g.set_titles(col_template="{col_name}", fontweight='bold', fontsize=18)
        g.fig.suptitle(ret_type + ' by sector')
        plt.figure()
        
    decile_count_sector = results.groupby([factor_1_name+" quantile", factor_2_name+" quantile", "Sector"])['Overnight_return'].count()
    
    def facet_heatmap_count(data, color, value_col, **kws):
            data = data.pivot(index=factor_1_name+" quantile", columns=factor_2_name+" quantile", values = value_col)
            sns.heatmap(data, annot=True, linewidths=.5, fmt='g', cmap="Blues", **kws)


    with sns.plotting_context(font_scale=5):
        g = sns.FacetGrid(decile_count_sector.reset_index(), col="Sector", col_wrap=3, size=4.5, aspect=1)

        g = g.map_dataframe(facet_heatmap_count, value_col = "Overnight_return")

        g.set_titles(col_template="{col_name}", fontweight='bold', fontsize=18)
        g.fig.suptitle('Number of observations by sector')
        plt.figure()
    
#     results['cum_intraday']=results.groupby(level = 1)['Intraday_return'].apply(lambda x : x.shift().fillna(0).add(1).cumprod()).values
#     results['cum_overnight']=results.groupby(level = 1)['Overnight_return'].apply(lambda x : x.shift().fillna(0).add(1).cumprod()).values
        
    return results

    

a = auto_heatmap(Fundamentals.market_cap, 
                 valuation_ratios.forward_earning_yield, "Pb Ratio", "Forward Earning Yield", 
                 5, '2018-01-01', '2019-01-01')

#In the Alphalens I am going to use Price to Book and Earnings Yield 
Price  Book: This is calculated as Price - Close (price_close_fp) divided by Book Value Per Share (bps).
Earnings Yield: This is calculated as Earnings Per Share - Fiscal Period (eps) divided by Price - Close (price_close_fp), multiplied by 100.

from quantopian.pipeline.data import factset

from quantopian.pipeline import Pipeline
from quantopian.research import run_pipeline
from quantopian.pipeline.filters import QTradableStocksUS

def make_pipeline():

    # Measures a company's asset growth rate.
    asset_growth = factset.Fundamentals.assets_gr_qf.latest 

    return Pipeline(
      columns={'asset_growth': asset_growth},
      screen=QTradableStocksUS() & asset_growth.notnull()
    )

factor_data = run_pipeline(pipeline=make_pipeline(), start_date='2006-1-1', end_date='2018-1-1')

# Show the first 10 rows of factor data
factor_data.head(10)

pricing_data = get_pricing(
  symbols=factor_data.index.levels[1], # Finds all assets that appear at least once in "factor_data"  
  start_date='2006-1-1',
  end_date='2018-2-1', # must be after run_pipeline()'s end date. Explained more in lesson 4
  fields='open_price' # Generally, you should use open pricing. Explained more in lesson 4
)

# Show the first 10 rows of pricing_data
pricing_data.head(10)

from alphalens.utils import get_clean_factor_and_forward_returns

merged_data = get_clean_factor_and_forward_returns(
  factor=factor_data, 
  prices=pricing_data
)

# Show the first 10 rows of merged_data
merged_data.head(10)

from quantopian.pipeline.data import factset

from quantopian.pipeline import Pipeline
from quantopian.research import run_pipeline
from quantopian.pipeline.factors import CustomFactor, SimpleMovingAverage
from quantopian.pipeline.filters import QTradableStocksUS
from alphalens.tears import create_information_tear_sheet
from alphalens.utils import get_clean_factor_and_forward_returns

def make_pipeline():

    # 1 year moving average of year over year net income
    net_income_moving_average = SimpleMovingAverage( 
      inputs=[factset.Fundamentals.net_inc_af], 
      window_length=252
    )

    # 1 year moving average of earnings yield
    earnings_yield_moving_average = SimpleMovingAverage( 
      inputs=[factset.Fundamentals.mkt_val], 
      window_length=252
    )

    average_earnings_yield_per_net_income = (earnings_yield_moving_average / net_income_moving_average)

    # the last quarter's net income
    net_income = factset.Fundamentals.net_inc_qf.latest 

    projected_earnings_yield = average_earnings_yield_per_net_income * net_income

    return Pipeline(
      columns={'projected_earnings_yield': projected_earnings_yield},
      screen=QTradableStocksUS() & projected_earnings_yield.notnull()
    )


factor_data = run_pipeline(make_pipeline(), '2006-1-1', '2018-1-1')
pricing_data = get_pricing(factor_data.index.levels[1], '2006-1-1', '2018-2-1', fields='open_price')
merged_data = get_clean_factor_and_forward_returns(factor_data, pricing_data)

create_information_tear_sheet(merged_data)

def make_pipeline():

    # 1 year moving average of year over year net income
    net_income_moving_average = SimpleMovingAverage( 
        inputs=[factset.Fundamentals.net_inc_af], 
        window_length=252
    )

    # 1 year moving average of earnings yield
    earnings_yield_moving_average = SimpleMovingAverage( 
        inputs=[factset.Fundamentals.mkt_val], 
        window_length=252
    )

    average_earnings_yield_per_net_income = (earnings_yield_moving_average / net_income_moving_average)

    net_income = factset.Fundamentals.net_inc_qf.latest # The last quarter's net income

    projected_earnings_yield = average_earnings_yield_per_net_income * net_income

    price_to_book = factset.Fundamentals.pbk_qf.latest # The alpha factor we are adding

    factor_to_analyze = projected_earnings_yield.zscore() + price_to_book.zscore()

    return Pipeline(
        columns={'factor_to_analyze': factor_to_analyze},
        screen=QTradableStocksUS() & factor_to_analyze.notnull()
    )



factor_data = run_pipeline(make_pipeline(), '2006-1-1', '2018-1-1')
pricing_data = get_pricing(factor_data.index.levels[1], '2006-1-1', '2018-1-1', fields='open_price')
new_merged_data = get_clean_factor_and_forward_returns(factor_data, pricing_data)

create_information_tear_sheet(new_merged_data)

from alphalens.tears import create_returns_tear_sheet

create_returns_tear_sheet(merged_data)

merged_data

import matplotlib.pyplot as plt
companies_per_day = merged_data.groupby(pd.Grouper(freq='D', level = 0)).count()
companies_per_day = companies_per_day[(companies_per_day.T != 0).any()]
plt.plot(companies_per_day['factor'])
plt.ylabel('Number of companies per day')
plt.show()

import seaborn as sns
sns.violinplot(x="factor_quantile", y="1D",
               split=True, inner="quart",
               data=merged_data)

merged_data

from alphalens.performance import create_pyfolio_input

import alphalens
import pyfolio

pf_returns, pf_positions, pf_benchmark = \
    create_pyfolio_input(merged_data,
                         period='1D',
                         capital=1000000,
                         long_short=True,
                         group_neutral=False,
                         equal_weight=False, # Equal weight vs weight based on alpha factor
                         quantiles=[3,4], # Choose the "best" quantiles to trade based on your analysis above
                         groups=None,
                         benchmark_period='1D') 

from pyfolio.tears import create_full_tear_sheet

create_full_tear_sheet(pf_returns,
                       positions=pf_positions,
                       benchmark_rets=pf_benchmark,
                       round_trips=True)