from quantopian.pipeline.data import quandl
from quantopian.pipeline.data.quandl import yahoo_index_vix

import datetime
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

frequency = 'daily'
def get_data(tickers,start,end):
    # get price
    p = get_pricing(tickers,start_date=start,
                    end_date = end,frequency=frequency)
    p = p.fillna(method='ffill')
    
    ret = pd.DataFrame(np.diff(np.log(p['price']),axis=0),columns=p.minor_axis,index=p.major_axis[1:])
    ret = ret.fillna(value=0)
    # get returns
    return p, ret

#context.stocks = [sid(24744), # RSP sid(22887), # EDV sid(23921), # TLT sid(40516)] # XIV   
#https://www.quantopian.com/posts/minimum-variance-w-slash-constraint Gant Kienhn, modified by Garyha, Tim Vidma, Peter Bakker                  
backtest = get_backtest('56bc74ed6aaabd12b9523d59') 
indv_p,indv_ret = get_data(symbols(['RSP','EDV','TLT','XIV']),backtest.start_date,backtest.end_date)

# equal weighted 
equal_weighted_ret = np.sum(np.diff(np.log(indv_p['price']),axis=0),axis=1)
equal_weighted_ret = np.cumsum(equal_weighted_ret)+1

# http://www.etfscreen.com/corrsym.php?s=SPY
etfs_all_str = ['SPY','IVV', 'SSO', 'UPRO', 'VV', 'SPXL', 'VOO', 'SCHX', 'IWB', 'ITOT',
       'SCHB', 'IWV', 'VTI', 'OEF', 'MGC', 'IYY', 'VONE', 'QUAL', 'PRF',
       'CSM', 'DGRW', 'DSI', 'DLN', 'MGV', 'JKD', 'XLG', 'VTV', 'SCHV',
       'DIA', 'IVW', 'DDM', 'VUG', 'UDOW', 'FNDX', 'DGRO', 'IVE', 'IWF',
       'RSP', 'IWD', 'FEX', 'RWL', 'VOOG', 'MGK', 'SPYG', 'IUSG', 'PKW',
       'VONG', 'KLD', 'IUSV', 'DTD', 'IWR', 'SCHD', 'ACWI', 'VIG', 'VYM',
       'PWV', 'IWY', 'FNDB', 'SCHG', 'SPYV', 'VT', 'VONV', 'VOOV', 'VO',
       'IOO', 'QDF', 'AOA', 'NOBL', 'PFM', 'TTFS', 'FBGX', 'IWP', 'OUSA',
       'JKG', 'RPG', 'USMV', 'FTC', 'JKE', 'SDY', 'SYLD', 'QDEF', 'KNOW',
       'TECL', 'QQEW', 'XLK', 'SCHM', 'FTCS', 'VOT', 'VOE', 'IWS', 'TILT',
       'AOR', 'FAS', 'FXO', 'IXN', 'JKF', 'IJH', 'ONEQ', 'RYT', 'IYF',
       'UYG', 'MDY', 'MVV', 'VGT', 'TQQQ', 'FTEC', 'IYJ', 'MIDU', 'QQQ',
       'QLD', 'DON', 'IGM', 'FVD', 'IVOO', 'QQQE', 'PDP', 'ACWV', 'DHS',
       'VFH', 'HDV', 'RYF', 'XLF', 'RXI', 'XT', 'SPLV', 'FNCL', 'IJK',
       'IYW', 'VTHR', 'ROM', 'QQXT', 'BFOR', 'PWB', 'IYC', 'FDIS', 'XLI',
       'MDYG', 'IVOG', 'IXG', 'SPHQ', 'CWB', 'VCR', 'VB', 'DTN', 'VXF',
       'VBR', 'XLY', 'RPV', 'TDIV', 'URTH', 'VIS', 'IYK', 'FXL', 'SKYY',
       'EZM', 'IJJ', 'FNX', 'FAD', 'FTLS', 'MTUM', 'FIDU', 'FNDA', 'DVY',
       'MOAT', 'FTA', 'IYG', 'FDL', 'QTEC', 'XMLV', 'EXI', 'SCHA', 'AOM',
       'PPA', 'FPX', 'FNY', 'VUSE', 'MDYV', 'FXD', 'VBK', 'KIE', 'RFG',
       'MTK', 'FIGY', 'PBP', 'SDOG', 'VLUE', 'HEFA', 'RCD', 'IJR', 'VEU',
       'SCHF', 'PEY', 'DBEF', 'SPHD', 'IAK', 'ITA', 'DBEU', 'RWK', 'IXUS',
       'DEF', 'PRFZ', 'CDC', 'IEFA', 'ACWX', 'VEA', 'EFA', 'DES', 'PSL',
       'NFRA', 'IWM', 'TNA', 'SLYG', 'URTY', 'CWI', 'SPHB', 'IGV', 'FLGE',
       'UWM', 'KXI', 'VXUS', 'DZK', 'EFG', 'FSTA', 'IXJ', 'GWL', 'DWM',
       'PBS', 'IWN', 'VIOO', 'VDC', 'FXR', 'EFV', 'CSD', 'FNDC', 'FFTY',
       'IJS', 'XLP', 'PXF', 'JPIN', 'DOL', 'VIOG', 'GAL', 'IAI', 'SLY',
       'SCZ', 'FNDF', 'EFAV', 'FDT', 'KBWB', 'SDIV', 'IHDG', 'HEZU', 'FV',
       'DBAW', 'SLYV', 'FEU', 'EUSC', 'IJT', 'PTF', 'SCHC', 'TLTD', 'PSP',
       'VSS', 'XAR', 'XSLV', 'IXP', 'VGK', 'SVXY', 'CGW', 'IEV', 'JKL',
       'CVY', 'XIV', 'RTH', 'DOO', 'IQDF', 'PHO', 'FCOM', 'MOO', 'HEDJ',
       'FDN', 'IWO', 'FYX', 'RWJ', 'RZG', 'FYC', 'RHS', 'DLS', 'VPL',
       'FGD', 'RWX', 'IYZ', 'PBJ', 'EWU', 'PIZ', 'PSCT', 'EES', 'FXG',
       'XHB', 'DVYL', 'VNQI', 'VOX', 'IYT', 'IDLV', 'GQRE', 'IAT', 'IFV',
       'PEZ', 'FEZ', 'VTWO', 'DTH', 'RWO', 'IDV', 'VTWG', 'IFGL', 'EZU',
       'WDIV', 'EWQ', 'HEWG', 'IGF', 'XLV', 'PNQI', 'GWX', 'DBEM', 'FDD',
       'DWAS', 'PSJ', 'SCHE', 'IEMG', 'DWX', 'DBJP', 'GMM', 'CURE', 'KBE',
       'KBWP', 'HEWJ', 'GMF', 'VTWV', 'DBGR', 'DBEZ', 'GUSH', 'VWO', 'EWN',
       'DXJ', 'IPAC', 'EDC', 'FXZ', 'FEP', 'EWJ', 'EUFN', 'EEM', 'RXL',
       'RYH', 'IYH', 'DGS', 'AOK', 'IGN', 'IHI', 'EEMV', 'VHT', 'FMAT',
       'BKF', 'DXGE', 'PID', 'XRT', 'PSCD', 'XLB', 'PEJ', 'EWX', 'HEEM',
       'VAW', 'AAXJ', 'FHLC', 'SOXL', 'MXI', 'SOXX', 'PDN', 'PPH', 'IEUR',
       'IWC', 'FNDE', 'ZIV', 'HACK', 'WOOD', 'DNL', 'ITB', 'URE', 'IYR',
       'SMH', 'FEM', 'DIV', 'RETL', 'IYM', 'GXC', 'ECON', 'VIDI', 'PIE',
       'SOCL', 'DEM', 'DFJ', 'FKU', 'PXH', 'MCHI', 'SCJ', 'KRE', 'TLTE',
       'JETS', 'EWG', 'EWW', 'FNI', 'XTN', 'INDY', 'AIA', 'REET', 'EWI',
       'DXJS', 'EWL', 'DXJR', 'INDL', 'EELV', 'EWC', 'IYLD', 'XSD', 'RZV',
       'EDIV', 'HAP', 'EWK', 'EEMA', 'GNR', 'XLE', 'INDA', 'EWD', 'IXC',
       'KBWD', 'FJP', 'XPP', 'ERX', 'VNQ', 'DVYE', 'YINN', 'FRI', 'EPHE',
       'DRN', 'PIN', 'EEMS', 'HDLV', 'GUNR', 'EPI', 'JXI', 'FXI', 'IYE',
       'MDIV', 'RWR', 'DFE', 'DIG', 'FENY', 'SCHH', 'EPP', 'VDE', 'ITF',
       'QABA', 'DVP', 'PJP', 'IGE', 'FXH', 'HEWY', 'EWP', 'ICF', 'INP',
       'IHE', 'ILF', 'FHK', 'EWT', 'BBH', 'PCEF', 'THD', 'JNK', 'EWY',
       'FSZ', 'IFEU', 'IEO', 'EWO', 'EWA', 'IBB', 'BIB', 'EWH', 'CEFL',
       'EIRL', 'DBKO', 'QAI', 'RYE', 'FGM', 'PSCH', 'INCO', 'EWS', 'DVHL',
       'SCIF', 'PSCC', 'FXU', 'LMLP', 'PICK', 'EIDO', 'IDX', 'HYG', 'REZ',
       'RYU', 'SMHD', 'PGJ', 'EZA', 'VNM', 'EMLP', 'KWEB', 'EDEN', 'EIS',
       'EWM', 'FBT', 'OIH', 'FXN', 'RUSL', 'IHF', 'RSX', 'UBIO', 'SJNK',
       'ECH', 'PBE', 'URA', 'PHB', 'ERUS', 'XPH', 'GYLD', 'IEZ', 'PFXF',
       'TAN', 'XHS', 'FEMS', 'EPU', 'VPU', 'ALFA', 'GREK', 'HYS', 'PTH',
       'MORT', 'IDU', 'XOP', 'FUTY', 'BRZU', 'SHYG', 'EWZ', 'MLPX', 'REM',
       'LEMB', 'FM', 'YCS', 'XLU', 'XBI', 'COMT', 'MORL', 'LABU', 'BSJJ',
       'BDCL', 'XES', 'FTRI', 'PFF', 'SBIO', 'XME', 'PEK', 'GXG', 'ELD',
       'EPOL', 'HYLS', 'HYHG', 'TUR', 'AMJ', 'AMU', 'ATMP', 'FPXI', 'ANGL',
       'EMB', 'BSJI', 'PUI', 'EFO', 'MLPI', 'BKLN', 'EMLC', 'PSCU', 'MLPN',
       'MLPA', 'ASHS', 'QEMM', 'IMLP', 'ENZL', 'USL', 'SPFF', 'CNXT',
       'DBC', 'DBO', 'BNO', 'GSG', 'DBE', 'HYLD', 'IGHG', 'GASL', 'PHDG',
       'GSP', 'PCY', 'FCG', 'GHYG', 'VWOB', 'RJI', 'UCO', 'UWTI', 'BSJH',
       'USO', 'DJP', 'AMLP', 'FXC', 'MLPL', 'TMV', 'EBND', 'TTT', 'TBT',
       'TBF', 'PST', 'PGF', 'HYEM', 'FXA', 'UGA', 'SRLN', 'FTGC', 'PGX',
       'PSK', 'TRSK', 'ASHR', 'EUO', 'GCC', 'BSJG', 'CHAU', 'UUP', 'PALL',
       'OIL', 'USCI', 'PDBC', 'RIGS', 'DBB', 'SIL', 'PPLT', 'VRP', 'JO',
       'SNLN', 'LQDH', 'KCNY', 'FPE', 'DBA', 'IHY', 'DSUM', 'GDXJ', 'GDX',
       'FLOT', 'JNUG', 'SGDM', 'NUGT', 'AGQ', 'SLV', 'USLV', 'BSCG',
       'EMCB', 'GLDX', 'FLRN', 'DZZ', 'FTSD', 'DGZ', 'GLL', 'DGLD', 'XMPT',
       'BSCH', 'AGGY', 'WBII', 'GLTR', 'USDU', 'FLTB', 'FTSL', 'BSCM',
       'SGG', 'FTSM', 'HYMB', 'JJG', 'CYB', 'VCSH', 'TDTT', 'CSJ', 'ILB',
       'GSY', 'HYD', 'BSCJ', 'FBND', 'DGAZ', 'MINC', 'SUB', 'KOLD', 'STPZ',
       'ISTB', 'FXB', 'VTIP', 'BSCI', 'BOIL', 'STIP', 'UGAZ', 'SHM',
       'JGBD', 'SCPB', 'UNG', 'ULST', 'SMB', 'BSCL', 'ITR', 'WIP', 'BIL',
       'PTLC', 'NEAR', 'LQD', 'CORP', 'CLY', 'BSCK', 'TDTF', 'BNDX', 'DGL',
       'UGLD', 'DGP', 'TIP', 'VCLT', 'GLD', 'UGL', 'SGOL', 'CIU', 'IAU',
       'CRED', 'JGBT', 'SLQD', 'SHV', 'VCIT', 'IPE', 'WDTI', 'ZSL', 'TOTL',
       'SCHP', 'LWC', 'DSLV', 'JDST', 'BOND', 'DUST', 'MINT', 'PZA',
       'LTPZ', 'ILTB', 'MUNI', 'IBDK', 'CMBS', 'BND', 'TFI', 'ITM', 'VBND',
       'IBDM', 'AGG', 'MBB', 'VMBS', 'SCHO', 'TUZ', 'MUB', 'SCHZ', 'BLV',
       'GGOV', 'AGZ', 'LAG', 'BAB', 'SHY', 'BIV', 'GVI', 'ITE', 'GBF',
       'BWX', 'BSV', 'CMF', 'UDN', 'IEI', 'FXE', 'UBT', 'VGSH', 'DTO',
       'IBND', 'EDV', 'VGLT', 'TLT', 'IEF', 'TLH', 'VGIT', 'TMF', 'SCHR',
       'ZROZ', 'TLO', 'FXF', 'GOVT', 'IGOV', 'UST', 'PLW', 'DWTI', 'DNO',
       'SCO', 'CHAD', 'BTAL', 'PTMC', 'LABD', 'FXY', 'BZQ', 'SJB', 'RUSS',
       'ZBIO', 'DDG', 'BIS', 'DUG', 'DRV', 'FXP', 'YANG', 'ERY', 'VXZ',
       'SMN', 'VIXM', 'SRS', 'SOXS', 'DRIP', 'EDZ', 'EEV', 'EUM', 'EWV',
       'EPV', 'VIIX', 'UVXY', 'VIXY', 'VXX', 'HDGE', 'TVIX', 'EFZ', 'RWM',
       'SRTY', 'TWM', 'TZA', 'SEF', 'SMDD', 'MIDZ', 'SQQQ', 'PSQ', 'QID',
       'SKF', 'TECS', 'FAZ', 'DOG', 'SDOW', 'DXD', 'SH', 'SPXU', 'SDS',
       'SPXS']

# get list of etfs within period of interest
def get_mature_etfs(etfs_all_str,start,end):    
    etfs_all = symbols(etfs_all_str)
    etfs_start_date = []
    etfs_end_date = []
    for n,r in enumerate(etfs_all):
        etfs_start_date.append(datetime.datetime.strptime(r.start_date.strip('GMT').split(',')[-1], ' %d %b %Y 00:00:00 '))
        etfs_end_date.append(datetime.datetime.strptime(r.end_date.strip('GMT').split(',')[-1], ' %d %b %Y 00:00:00 '))    
    etfs = [x for x,y,z in zip(etfs_all,etfs_start_date,etfs_end_date) if y.date() <= start and z.date() > end]
    print(len(etfs_all),len(etfs))
    return etfs

# initialize params
frequency = 'daily'
# get recent & surviving etfs
etfs = get_mature_etfs(etfs_all_str,backtest.start_date.date(),backtest.end_date.date())

_p,_ret = get_data(etfs,backtest.start_date.date(),backtest.end_date.date())

get_ind = lambda arr,tick: [np.argwhere(np.array(arr)==x)[0,0] for x in symbols([tick])][0]

p=np.copy(_p)
ret=_ret.copy()
# down_market 150:250, 550:650, 900:1050, 1150:1300 
# up_market 0:150, 300:500, 700:900
down_ret = np.concatenate([_ret.iloc[150:250,:],_ret.iloc[550:650,:],_ret.iloc[900:1050,:],_ret.iloc[1150:1300,:]])
up_ret = np.concatenate([_ret.iloc[0:150,:],_ret.iloc[300:500,:],_ret.iloc[700:900,:]])

def get_score(ret,std_impact=1):
    ret = ret.as_matrix()
    mean_arr = np.mean(ret,axis=0)
    std_arr = np.std(ret,axis=0)
    adj_mean = mean_arr/std_arr**std_impact
    return adj_mean

def visualize_data(ret,get_score,lim,std_impact=1,benchmark=202):
    
    score = get_score(ret,std_impact)
    print(score.shape)
    plt.subplot(221)
    plt.hist(score[~np.isnan(score)])
    print(get_ind(etfs,'XIV'),score[get_ind(etfs,'XIV')])
    lim_p=np.percentile(score,lim)
    print('lim',lim)
    inds = np.argwhere([score>lim_p])[:,1]
    print('# that is > lim',len(inds))

    plt.subplot(222)
    for row in inds:
        a=plt.plot(np.cumsum(ret[etfs[row]]),label=str(row))
    if benchmark:
        a=plt.plot(np.cumsum(ret[etfs[benchmark]]),color='black',linestyle='--',label=etfs[benchmark].symbol)
    plt.legend(loc=0)
    
    return inds

print(down_ret.shape)
inds = visualize_data(pd.DataFrame(down_ret,columns=ret.columns),get_score,98,std_impact=1)
print("best performing stocks - bull market")

for r in inds: 
    print(r,etfs[r].sid,etfs[r].symbol,etfs[r].asset_name)

inds=visualize_data(pd.DataFrame(up_ret,columns=ret.columns),get_score,98,std_impact=0.5,benchmark=None)
print("best performing stocks - bear market")

for r in inds:
    print(r,etfs[r].sid,etfs[r].symbol,etfs[r].asset_name)

inds= visualize_data(pd.DataFrame(ret,columns=ret.columns),get_score,99,std_impact=0.7,benchmark=0)
print("best performing stocks - overall market")

for r in inds:
    print(r,etfs[r].sid,etfs[r].symbol,etfs[r].asset_name)

inds= visualize_data(pd.DataFrame(ret,columns=ret.columns),get_score,99,std_impact=0.5,benchmark=0)
print("best performing stocks - overall market")
for r in inds:
    print(r,etfs[r].sid,etfs[r].symbol,etfs[r].asset_name)

for r in inds:
    print(r,etfs[r].sid,etfs[r].symbol,etfs[r].asset_name)

#(87, 39214, u'TQQQ', u'PROSHARES ULTRAPRO QQQ                  ')
# (288, 39901, u'RETL', u'DIREXION DAILY RETAIL BULL 3X')
# (177, 25903, u'VDC', u'VANGUARD CONSUMER STAPLES ETF')
# (182, 19659, u'XLP', u'CONSUMER STAPLES SELECT SECTOR SPDR FUND')
# (212, 32841, u'RHS', u'GUGGENHEIM S&P 500 EQUAL WEIGH')
# (223, 33798, u'FXG', u'FIRST TRUST CONSUMER STAPLES ALPHADEX FUND')
# (506, 39986, u'SCHO', u'SCHWAB SHORT-TERM U.S. TREASURY ETF')
# (512, 23911, u'SHY', u'ISHARES 1-3 YEAR TREASURY BOND')
#(212, 32841, u'RHS', u'GUGGENHEIM S&P 500 EQUAL WEIGH')
# (527, 22887, u'EDV', u'VANGUARD EXTENDED DURATION TREASURY ETF')
# (528, 38988, u'VGLT', u'VANGUARD LONG-TERM GOVERNMENT BOND ETF')
# (529, 23921, u'TLT', u'ISHARES 20+ YEAR TREASURY BOND')
# (535, 38842, u'ZROZ', u'PIMCO 25+ YEAR ZERO COUPON U.S')

tickers = symbols(['VDC','ZROZ','RETL','SHY','TQQQ'])
_,indv_ret = get_data(tickers,backtest.start_date,backtest.end_date)

np.cumsum(indv_ret).plot()

plt.subplot(221)
plt.imshow(np.corrcoef(indv_ret.T),cmap='jet',interpolation='none')
plt.colorbar()

backtest = get_backtest('56c5742a7e4d950de805586a') 
indv_p,indv_ret = get_data(symbols(['VDC','ZROZ','RETL','VGLT','SHY','TQQQ']),backtest.start_date,backtest.end_date)

plt.subplot(221)
plt.plot(indv_p['price']/indv_p['price'].iloc[0,:])
plt.plot(backtest.cumulative_performance.returns.as_matrix()+1,color='black',linewidth=2,label='strategy')
plt.legend(loc=2)

# basics
#  https://www.math.ust.hk/~maykwok/courses/ma362/Topic2.pdf
# variance estimate
#? http://papers.ssrn.com/sol3/papers.cfm?abstract_id=2302453
# use > 10? uncorrelated return streams
#  https://www.quantopian.com/posts/monte-carlo-simulations-of-correlated-portfolios-the-quest-for-uncorrelated-return-streams
# http://nbviewer.jupyter.org/github/paulperry/quant/blob/master/OLPS_Comparison.ipynb