"""
To Do
- Dynamic selection of ETFs vs static in Risk Manamgent section.  Line 68?
- Volatility Position Sizing of Index ETF Basket?  See Clenow formula...https://www.followingthetrend.com/2017/06/volatility-parity-position-sizing-using-standard-deviation/
- Post trades, current cash available/margin used, and current postions and results site for Team / stakeholders to view via web.
- Post portfolio changes and current allocation to private area on Agile side (Signal subscription for other advisors / institutions not on IB)
- Execution options (Market vs Limit vs VWAP)  https://github.com/QuantConnect/Lean/tree/master/Algorithm.Framework/Execution
- Publish Reports to Website (Where users can register to see results)
    > Jim's Comments - Change the word backtest to strategy in "the report"
- Logging and live trade review/reporting.

MUCH LATER >. can this approach be modified to use for futures?

Better to use unlevered ETFs in the Risk Management and add leverage to the account at IB.  Illiquidity and float issues, not to mention volatility difference in underlying.
"""

from QuantConnect.Data.UniverseSelection import *
import numpy as np
import pandas as pd
# --------------------------------------------------------------------------------------------------------
BONDS = ['TLT','GLD']; VOLA = 126; BASE_RET = 85; STK_MOM = 126; N_COARSE = 100; N_FACTOR = 20; N_MOM = 20; LEV = .99; 
# --------------------------------------------------------------------------------------------------------

class Fundamental_Factors_Momentum_ROC_Comparison_OUT_DAY(QCAlgorithm):

    def Initialize(self):

        # LIVE TRADING
        if self.LiveMode:
            self.Debug("Trading Live!")
        self.SetBrokerageModel(BrokerageName.InteractiveBrokersBrokerage, AccountType.Margin)
        
        # Group Trading
        # Use a default FA Account Group with an Allocation Method
        self.DefaultOrderProperties = InteractiveBrokersOrderProperties()
        
        # account group created manually in IB/TWS
        self.DefaultOrderProperties.FaGroup = "TE"

        # supported allocation methods are: EqualQuantity, NetLiq, AvailableEquity, PctChange
        self.DefaultOrderProperties.FaMethod = "AvailableEquity"

        # set a default FA Allocation Profile
        # Alex: I commented the following line out, since it would "reset" the previous settings
        #self.DefaultOrderProperties = InteractiveBrokersOrderProperties()

        # allocation profile created manually in IB/TWS
        # self.DefaultOrderProperties.FaProfile = "TestProfileP"
        
        #Algo Start
        self.SetStartDate(2015, 1, 1)  
        #self.SetEndDate(2010, 12, 31)  
        self.InitCash = 100000
        self.SetCash(self.InitCash)  
        self.MKT = self.AddEquity("SPY", Resolution.Hour).Symbol
        self.mkt = []
        self.SetBrokerageModel(BrokerageName.InteractiveBrokersBrokerage, AccountType.Margin)
        res = Resolution.Hour        

        self.BONDS = [self.AddEquity(ticker, res).Symbol for ticker in BONDS]
        self.INI_WAIT_DAYS = 5  
        self.wait_days = self.INI_WAIT_DAYS        

        self.GLD = self.AddEquity('GLD', res).Symbol
        self.SLV = self.AddEquity('SLV', res).Symbol
        self.XLU = self.AddEquity('XLU', res).Symbol
        self.XLI = self.AddEquity('XLI', res).Symbol
        self.UUP = self.AddEquity('UUP', res).Symbol
        self.DBB = self.AddEquity('DBB', res).Symbol
        
        self.pairs = [self.GLD, self.SLV, self.XLU, self.XLI, self.UUP, self.DBB]

        self.bull = 1
        self.bull_prior = 0
        self.count = 0 
        self.outday = (-self.INI_WAIT_DAYS+1)
        self.SetWarmUp(timedelta(350))

        self.UniverseSettings.Resolution = res
        self.AddUniverse(self.CoarseFilter, self.FineFilter)
        self.data = {}
        self.RebalanceFreq = 10 
        self.UpdateFineFilter = 0
        self.symbols = None
        self.RebalanceCount = 0
        self.wt = {}
        
        self.Schedule.On(self.DateRules.EveryDay(), self.TimeRules.AfterMarketOpen('SPY', 180), 
            self.daily_check)
        
        symbols = [self.MKT] + self.pairs
        for symbol in symbols:
            self.consolidator = TradeBarConsolidator(timedelta(days=1))
            self.consolidator.DataConsolidated += self.consolidation_handler
            self.SubscriptionManager.AddConsolidator(symbol, self.consolidator)
        
        self.history = self.History(symbols, VOLA, Resolution.Daily)
        if self.history.empty or 'close' not in self.history.columns:
            return
        self.history = self.history['close'].unstack(level=0).dropna()
        
    def consolidation_handler(self, sender, consolidated):
        self.history.loc[consolidated.EndTime, consolidated.Symbol] = consolidated.Close
        self.history = self.history.iloc[-VOLA:]
        
    def derive_vola_waitdays(self):
        sigma = 0.6 * np.log1p(self.history[[self.MKT]].pct_change()).std() * np.sqrt(252)
        wait_days = int(sigma * BASE_RET)
        period = int((1.0 - sigma) * BASE_RET)
        return wait_days, period       
        
        
    def CoarseFilter(self, coarse):
        
        if not (((self.count-self.RebalanceCount) == self.RebalanceFreq) or (self.count == self.outday + self.wait_days - 1)):
            self.UpdateFineFilter = 0
            return Universe.Unchanged
        
        self.UpdateFineFilter = 1

        selected = [x for x in coarse if (x.HasFundamentalData) and (float(x.Price) > 5)]
        filtered = sorted(selected, key=lambda x: x.DollarVolume, reverse=True)
        return [x.Symbol for x in filtered[:N_COARSE]]
        
        
    def FineFilter(self, fundamental):
        if self.UpdateFineFilter == 0:
            return Universe.Unchanged
            
        filtered_fundamental = [x for x in fundamental if (x.ValuationRatios.EVToEBITDA > 0) 
                                        and (x.EarningReports.BasicAverageShares.ThreeMonths > 0) 
                                        and float(x.EarningReports.BasicAverageShares.ThreeMonths) * x.Price > 50e9
                                        and x.SecurityReference.IsPrimaryShare
                                        and x.SecurityReference.SecurityType == "ST00000001"
                                        and x.SecurityReference.IsDepositaryReceipt == 0
                                        and x.CompanyReference.IsLimitedPartnership == 0]

        top = sorted(filtered_fundamental, key = lambda x: x.ValuationRatios.EVToEBITDA, reverse=True)[:N_FACTOR]
        self.symbols = [x.Symbol for x in top]
        self.UpdateFineFilter = 0
        self.RebalanceCount = self.count
        return self.symbols

    
    def OnSecuritiesChanged(self, changes):
   
        addedSymbols = []
        for security in changes.AddedSecurities:
            addedSymbols.append(security.Symbol)
            if security.Symbol not in self.data:
                self.data[security.Symbol] = SymbolData(security.Symbol, STK_MOM, self)
   
        if len(addedSymbols) > 0:
            history = self.History(addedSymbols, 1 + STK_MOM, Resolution.Daily).loc[addedSymbols]
            for symbol in addedSymbols:
                try:
                    self.data[symbol].Warmup(history.loc[symbol])
                except:
                    self.Debug(str(symbol))
                    continue
 
        
    def daily_check(self):
        self.wait_days, period = self.derive_vola_waitdays()

        r = self.history.pct_change(period).iloc[-1]
        
        bear = ((r[self.SLV] < r[self.GLD]) and (r[self.XLI] < r[self.XLU]) and (r[self.DBB] < r[self.UUP]))

        if bear:
            self.bull = False
            self.outday = self.count
            
        if (self.count >= self.outday + self.wait_days):
            self.bull = True
            
        self.wt_stk = LEV if self.bull else 0  
        self.wt_bnd = 0 if self.bull else LEV    
        
        if bear:
            self.trade_out()
       
        if (self.bull and not self.bull_prior) or (self.bull and (self.count==self.RebalanceCount)):
            self.trade_in()
                
        self.bull_prior = self.bull
        self.count += 1
        
        
    def trade_out(self):
        for sec in self.BONDS:
            self.wt[sec] = self.wt_bnd/len(self.BONDS)
        
        for sec in self.Portfolio.Keys:
            if sec not in self.BONDS:
                self.wt[sec] = 0
            
        for sec, weight in self.wt.items():
            if weight == 0 and self.Portfolio[sec].IsLong:
                self.Liquidate(sec)
                
        for sec, weight in self.wt.items(): 
            if weight != 0:
                self.SetHoldings(sec, weight)  


    def trade_in(self):
            
        if self.symbols is None: return
        
        output = self.calc_return(self.symbols)
        stocks = output.iloc[:N_MOM].index
        
        for sec in self.Portfolio.Keys:
            if sec not in stocks:                
                self.wt[sec] = 0
        for sec in stocks:
            self.wt[sec] = self.wt_stk/N_MOM
            
        for sec, weight in self.wt.items():             
            self.SetHoldings(sec, weight) 
 
        
    def calc_return(self, stocks):

        ret = {}
        for symbol in stocks:
            try:
                ret[symbol] = self.data[symbol].Roc.Current.Value
            except:
                self.Debug(str(symbol))
                continue
            
        df_ret = pd.DataFrame.from_dict(ret, orient='index')
        df_ret.columns = ['return']
        sort_return = df_ret.sort_values(by = ['return'], ascending = False)
        
        return sort_return
    
        
    def OnEndOfDay(self): 
        
        mkt_price = self.Securities[self.MKT].Close
        self.mkt.append(mkt_price)
        mkt_perf = self.InitCash * self.mkt[-1] / self.mkt[0] 
        self.Plot('Strategy Equity', self.MKT, mkt_perf)     
        
        account_leverage = self.Portfolio.TotalHoldingsValue / self.Portfolio.TotalPortfolioValue
        
        self.Plot('Holdings', 'leverage', round(account_leverage, 2))
        self.Plot('Holdings', 'Target Leverage', LEV)
    
        
class SymbolData(object):

    def __init__(self, symbol, roc, algorithm):
        self.Symbol = symbol
        self.Roc = RateOfChange(roc)
        self.algorithm = algorithm
        
        self.consolidator = algorithm.ResolveConsolidator(symbol, Resolution.Daily)
        algorithm.RegisterIndicator(symbol, self.Roc, self.consolidator)
        
    def Warmup(self, history):
        for index, row in history.iterrows():
            self.Roc.Update(index, row['close'])
            

"""
Intersection of ROC comparison using OUT_DAY approach by Vladimir v1.3 
(with dynamic selector for fundamental factors and momentum)

inspired by Peter Guenther, Tentor Testivis, Dan Whitnable, Thomas Chang, Miko M, Leandro Maia

Leandro Maia setup modified by Vladimir
https://www.quantconnect.com/forum/discussion/9632/amazing-returns-superior-stock-selection-strategy-superior-in-amp-out-strategy/p2/comment-29437
https://www.quantconnect.com/forum/discussion/10246/intersection-of-roc-comparison-using-out-day-approach/p1
BONDS = symbols('TMF') if data.can_trade(symbol('TMF')) else symbols('TLT')
"""