from quantopian.pipeline import Pipeline
from quantopian.pipeline.data import morningstar
from quantopian.pipeline.data.builtin import USEquityPricing
from quantopian.pipeline.factors import SimpleMovingAverage
from quantopian.pipeline.filters.morningstar import IsPrimaryShare
from quantopian.pipeline.data.zacks import broker_ratings
from quantopian.research import run_pipeline

from quantopian.pipeline.filters import Q1500US

from zipline.api import get_datetime
# from zipline.pipeline.factors.Factor import isnan

import numpy as np
import pandas as pd

def universe_filters():
    """
    Create a Pipeline producing Filters implementing common acceptance criteria.
    
    Returns
    -------
    zipline.Filter
        Filter to control tradeablility
    """
    # Primary share equities.
    primary_share = IsPrimaryShare()
    
    # Equities listed as common stock. 'ST00000001' indicates common stock.
    common_stock = morningstar.share_class_reference.security_type.latest.eq('ST00000001') 
    
    # Non-depositary receipts
    not_depositary = ~morningstar.share_class_reference.is_depositary_receipt.latest
    
    # Equities not trading over-the-counter.
    not_otc = ~morningstar.share_class_reference.exchange_id.latest.startswith('OTC')
    
    # Not when-issued equities.
    not_wi = ~morningstar.share_class_reference.symbol.latest.endswith('.WI')
    
    # Equities without LP in their name
    not_lp_name = ~morningstar.company_reference.standard_name.latest.matches('.* L[. ]?P.?$')
    
    # Equities with a null value in the limited_partnership
    not_lp_balance_sheet = morningstar.balance_sheet.limited_partnership.latest.isnull()
    
    # Equities whose most recent Morningstar market cap is not null have fundamental data and therefore are not ETFs.
    have_market_cap = morningstar.valuation.market_cap.latest.notnull()

    universe_filter = (
        primary_share
        & common_stock
        & not_depositary
        & not_otc
        & not_wi
        & not_lp_name
        & not_lp_balance_sheet
        & have_market_cap
    )
    
    return universe_filter

def make_pipeline():    
    #Make our universe filter 
    q1500us = Q1500US()
    
    #Get price data
    latest_close = USEquityPricing.close.latest
    mean_close_10 = SimpleMovingAverage(inputs=[USEquityPricing.close], window_length=10, mask=q1500us)
    percent_difference = latest_close / mean_close_10 - 1
    
    #Get key financial ratios
    price_to_book = morningstar.valuation_ratios.pb_ratio.latest #TO-DO: adjust for goodwill_and_other_intangible_assets
    price_to_earnings = morningstar.valuation_ratios.pe_ratio.latest  #TO-DO: exclude extraordinary earnings items
    debt_to_equity = morningstar.operation_ratios.total_debt_equity_ratio.latest
    financials_ratio = (1/price_to_book + 10/price_to_earnings - debt_to_equity)/3
    
    #Get last filing date
    last_filing_date = morningstar.financial_statement_filing.file_date.latest #TO-DO: check what is the most relevant statement date
    
    #Get analyst estimations
    rating_strong_buys = broker_ratings.rating_cnt_strong_buys.latest
    rating_buys = broker_ratings.rating_cnt_mod_buys.latest
    rating_holds = broker_ratings.rating_cnt_holds.latest
    rating_sells = broker_ratings.rating_cnt_mod_sells.latest
    rating_strong_sells = broker_ratings.rating_cnt_strong_sells.latest
        
    #Get Market capitalisation component
    market_cap = morningstar.valuation.market_cap.latest
    market_cap_coefficient = 46066000/market_cap

    return Pipeline(
        columns={
            'P/B': price_to_book,
            'P/E' : price_to_earnings,
            'D/E' : debt_to_equity,
            'Financials_ratio' : financials_ratio,
            'Last_filing':last_filing_date,
            'Strong_buys':rating_strong_buys, 
            'Buys':rating_buys, 
            'Holds':rating_holds, 
            'Sells':rating_sells, 
            'Strong_sells':rating_strong_sells,
            'Market_cap_coefficient': market_cap_coefficient,

        },
        # screen=universe_filters() & screening
        screen=q1500us,
    )

my_pipe = make_pipeline()

result = run_pipeline(my_pipe, '2013-05-05', '2013-05-05')
print 'Number of securities that passed the filter: %d' % len(result)

result

# The results dataframe contains all the raw data (and some simple calculated data) we need for our calculations
# We can clean up specific columns or the whole dataframe to get rid of NaNs or other odd data
# Here all the NaNs are replaced by zeros but this could be done more differently for each column
# The inplace parameter is set to True so we don't get another whole copy of the dataframe

result.fillna(0.0, inplace=True)

# Let's see what we have now
result

# The data doesn't have any NaNs now (assuming that's what we want)
# Let's add a rating_coefficient column
# This is a very straight forward way but could also be done by creating a dict of weights and multiply the entire dataframe

result['Rating_coefficient'] = result['Strong_buys'] * .025 \
                             + result['Buys'] * .015  \
                             + result['Holds'] * .005  \
                             + result['Sells'] * -.005 \
                             + result['Strong_sells'] * -.015
                    
# While we're at it lets calculate the potential and put that value into another column
    
result['Potential'] = result['Financials_ratio'] + result['Rating_coefficient'] - result['Market_cap_coefficient']

result

# Now lets create a new dataframe with just the top 25 highest potential securities and call it my_buys

my_buys = result.sort('Potential', ascending = False).head(25)

my_buys