from math import ceil
import numpy as np
import pandas as pd
import scipy as sp

### <summary>
### Demonstration of how to estimate constituents of QC500 index based on the company fundamentals
### The algorithm creates a default tradable and liquid universe containing 500 US equities
### which are chosen at the first trading day of each month.
### </summary>
class ConstituentsQC500GeneratorAlgorithm(QCAlgorithm):

    def Initialize(self):
        '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''

        self.SetStartDate(2012, 1, 1) #Set Start Date
        self.SetEndDate(2018, 9, 1) #Set End Date
        self.SetCash(100000) #Set Strategy Cash
        self.UniverseSettings.Resolution = Resolution.Daily

        # this add universe method accepts two parameters:
        # - coarse selection function: accepts an IEnumerable<CoarseFundamental> and returns an IEnumerable<Symbol>
        # - fine selection function: accepts an IEnumerable<FineFundamental> and returns an IEnumerable<Symbol>
        self.AddUniverse(self.CoarseSelectionFunction, self.FineSelectionFunction)

        self.spy = self.AddEquity("SPY", Resolution.Daily)
        self.Schedule.On(self.DateRules.MonthStart("SPY"), self.TimeRules.At(0, 0), self.monthly_rebalance)
        self.num_coarse = 1000
        self.num_fine = 500
        self.num_short_term = 60
        self.num_long_term = 30
        self.dollar_volume = {}
        self.rebalance = True
        self.ready = False
        self.symbolDataDict = {}
        
        self.short_term_indicator = 252 * 3
        self.long_term_indicator = 252 * 10

    def CoarseSelectionFunction(self, coarse):
        if not self.rebalance: return []
        # The stocks must have fundamental data
        # The stock must have positive previous-day close price
        # The stock must have positive volume on the previous trading day
        filtered = [x for x in coarse if x.HasFundamentalData
                                      and x.Volume > 0
                                      and x.Price > 0]
        # sort the stocks by dollar volume and take the top 1000
        sort_filtered = sorted(filtered, key=lambda x: x.DollarVolume, reverse=True)[:self.num_coarse]
        for i in sort_filtered:
            self.dollar_volume[i.Symbol.Value] = i.DollarVolume

        # return the symbol objects our sorted collection
        return [x.Symbol for x in sort_filtered]

    def FineSelectionFunction(self, fine):
        if not self.rebalance: return []
        # The company's headquarter must in the U.S.
        # The stock must be traded on either the NYSE or NASDAQ
        # At least half a year since its initial public offering
        # The stock's market cap must be greater than 500 million
        filtered_fine = [x for x in fine if  (x.CompanyReference.CountryId == "USA")
                                        and (x.CompanyReference.PrimaryExchangeID == "NYS" or x.CompanyReference.PrimaryExchangeID == "NAS")
                                        and ((self.Time - x.SecurityReference.IPODate).days > 180)
                                        and x.EarningReports.BasicAverageShares.ThreeMonths * (x.EarningReports.BasicEPS.TwelveMonths*x.ValuationRatios.PERatio) > 5e8]

        count = len(filtered_fine)
        if count == 0: return []
        
        # select stocks with top dollar volume in every single sector
        for i in filtered_fine:
            i.DollarVolume = self.dollar_volume[i.Symbol.Value]
        percent = float(self.num_fine/count)
        group_by_code = {}
        top_list = []
        for code in ["N", "M", "U", "T", "B", "I"]:
            group_by_code[code] = list(filter(lambda x: x.CompanyReference.IndustryTemplateCode == code, filtered_fine))
            top = sorted(group_by_code[code], key=lambda x: x.DollarVolume, reverse = True)[:ceil(len(group_by_code[code])*percent)]
            top_list.append(top)
        joined_list = top_list[0]
        for ls in top_list[1:]:
            joined_list  += ls
        self.symbols = [x.Symbol for x in joined_list][:self.num_fine]
        self.Log(",".join(sorted(i.Value for i in self.symbols)))
        self.ready = True
        return self.symbols

    def OnData(self, data):
        pass

    def monthly_rebalance(self):
        now = self.Time
        if now.month == 1:
            self.rebalance = True
            self.ready = False
        
    def OnData(self, data):
        if self.ready:
            self.ready = False
            self.rebalance = False

            for symbol, symbolData in self.symbolDataDict.items():
                # update the indicator value for securities already in the portfolio
                if symbol not in self.addedSymbols:
                    symbolData.MOMST.Update(IndicatorDataPoint(symbol, self.Time, self.Securities[symbol].Close))
                # liquidate removed securities
                if symbol in self.removedSymbols:
                    self.Liquidate(symbol)
                
            self.addedSymbols = []
            self.removedSymbols = []
            
            sorted_symbolData = sorted(self.symbolDataDict, key=lambda x: self.symbolDataDict[x].MOMST.Current.Value)
            long_stocks = sorted_symbolData[:60]
            
            for symbol in long_stocks:
                self.symbolDataDict[symbol].MOMLT.Update(IndicatorDataPoint(symbol, self.Time, self.Securities[symbol].Close))
            
            sorted_symbolData2 = sorted(long_stocks, key=lambda x: self.symbolDataDict[x].MOMLT.Current.Value)
            buy_stocks = sorted_symbolData2[:30]

            for long_stock in buy_stocks:
                self.SetHoldings(long_stock, 1/len(buy_stocks))
            stocks_invested = [x.Key for x in self.Portfolio if x.Value.Invested]
            # liquidate stocks not in the list 
            for i in stocks_invested:
                if i not in buy_stocks:
                    self.Liquidate(i)


    def OnSecuritiesChanged(self, changes):
        # clean up data for removed securities
        self.removedSymbols = [x.Symbol for x in changes.RemovedSecurities]
        for removed in changes.RemovedSecurities:
            symbolData = self.symbolDataDict.pop(removed.Symbol, None)
        # warm up the indicator with history price for newly added securities
        self.addedSymbols = [x.Symbol for x in changes.AddedSecurities if x.Symbol.Value != "SPY"]
        history = self.History(self.addedSymbols, self.long_term_indicator+1, Resolution.Daily)

        for symbol in self.addedSymbols:
            if symbol not in self.symbolDataDict.keys():
                symbolData = SymbolData(symbol, self.short_term_indicator, self.long_term_indicator)
                self.symbolDataDict[symbol] = symbolData
                if str(symbol) in history.index:             
                    symbolData.WarmUpIndicator(history.loc[str(symbol)])
            
class SymbolData:
    '''Contains data specific to a symbol required by this model'''
    
    def __init__(self, symbol, lookback, lookbacklong):
        self.symbol = symbol
        self.MOMST = Momentum(lookback)
        self.MOMLT = Momentum(lookbacklong)

    def WarmUpIndicator(self, history):
        # warm up the Momentum indicator with the history request
        for tuple in history.itertuples():
            item = IndicatorDataPoint(self.symbol, tuple.Index, float(tuple.close))
            self.MOMST.Update(item) 
            self.MOMLT.Update(item)