# O'Shaugnessey VC2 + Momentum
# PBRatio (Price-To-Book)
# PERatio (Price-To-Earnings)
# PSRatio (Price-To-Sales)
# EVToEBITDA
# PCFRatio (Price-To-Cash Flow)
# Absicherung nach unten durch SPY-SMA200-Filter
# Rebalancing Monthly

from QuantConnect.Data.UniverseSelection import *
import math
import numpy as np
import pandas as pd
import scipy as sp
# import statsmodels.api as sm

class FundamentalFactorAlgorithm(QCAlgorithm):

    def Initialize(self):

        self.SetStartDate(2000, 1, 1)
        self.SetCash(100000)          
  
        self.UniverseSettings.Resolution = Resolution.Daily
        self.AddUniverse(self.CoarseSelectionFunction, self.FineSelectionFunction)
        self.spy = self.AddEquity("SPY", Resolution.Minute).Symbol 
        self.holding_months = 1
        self.num_screener = 100
        self.num_stocks = 9
        self.formation_days = 200
        self.lowmom = False
        self.month_count = self.holding_months
        self.Schedule.On(self.DateRules.MonthStart("SPY"), self.TimeRules.At(0, 0), Action(self.monthly_rebalance))
        self.Schedule.On(self.DateRules.MonthStart("SPY"), self.TimeRules.At(10, 0), Action(self.rebalance))
        # rebalance the universe selection once a month
        self.rebalence_flag = 0
        # make sure to run the universe selection at the start of the algorithm even it's not the manth start
        self.first_month_trade_flag = 1
        self.trade_flag = 0 
        self.symbols = None
 
    def CoarseSelectionFunction(self, coarse):
        if self.rebalence_flag or self.first_month_trade_flag:
            selected = [x for x in coarse if (x.HasFundamentalData) and (x.Price > 5)]
            # rank the stocks by dollar volume 
            filtered = sorted(selected, key=lambda x: x.DollarVolume, reverse=True) 
    
            return [ x.Symbol for x in filtered[:800]] # 200
        else:
            return self.symbols

    def FineSelectionFunction(self, fine):
        if self.rebalence_flag or self.first_month_trade_flag:
            hist = self.History([i.Symbol for i in fine], 1, Resolution.Daily)

            try:
                filtered_fine = [x for x in fine if (x.ValuationRatios.EVToEBITDA > 0) 
                                                    and (x.ValuationRatios.PBRatio > 0)
                                                    and (x.ValuationRatios.PERatio > 0)
                                                    and (x.ValuationRatios.PCFRatio > 0)
                                                    and (x.ValuationRatios.TotalYield > 0)
                                                    and (x.EarningReports.BasicAverageShares.ThreeMonths > 0) 
                                                    and (x.ValuationRatios.PSRatio > 0)
                                                    and float(x.EarningReports.BasicAverageShares.ThreeMonths) * hist.loc[str(x.Symbol)]['close'][0] > 2e9]
            except:
                filtered_fine = [x for x in fine if (x.ValuationRatios.EVToEBITDA > 0) 
                                                and (x.ValuationRatios.PBRatio > 0)
                                                and (x.ValuationRatios.PERatio > 0)
                                                and (x.ValuationRatios.PCFRatio > 0)
                                                and (x.ValuationRatios.TotalYield > 0)
                                                and (x.EarningReports.BasicAverageShares.ThreeMonths > 0)
                                                and (x.ValuationRatios.PSRatio > 0)] 

            sortedByEVToEBITDA = sorted(filtered_fine, key = lambda x: x.ValuationRatios.EVToEBITDA, reverse=True)
            sortedByPSRatio = sorted(filtered_fine, key = lambda x: x.ValuationRatios.PSRatio, reverse=True)
            sortedByPBRatio = sorted(filtered_fine, key = lambda x: x.ValuationRatios.PBRatio, reverse=True)
            sortedByPERatio = sorted(filtered_fine, key = lambda x: x.ValuationRatios.PERatio, reverse=True)
            sortedByPCFRatio = sorted(filtered_fine, key = lambda x: x.ValuationRatios.PCFRatio, reverse=True)
            sortedByTotalYield = sorted(filtered_fine, key = lambda x: x.ValuationRatios.TotalYield, reverse=False)
            
            stock_dict = {}
            
            for rank1, ele in enumerate(sortedByEVToEBITDA):
                rank2 = sortedByPSRatio.index(ele)
                rank3 = sortedByPBRatio.index(ele)
                rank4 = sortedByPERatio.index(ele)
                rank5 = sortedByPCFRatio.index(ele)
                rank6 = sortedByTotalYield.index(ele)
                stock_dict[ele] = rank1 + rank2 + rank3 + rank4 + rank5 + rank6
            
            top = sorted(stock_dict.items(), key=lambda d:d[1], reverse=True)[:self.num_screener]
            
    #        top = sorted(filtered_fine, key = lambda x: x.ValuationRatios.EVToEBITDA, reverse=True)[:self.num_screener]
            self.symbols = [x[0].Symbol for x in top]
            
            self.rebalence_flag = 0
            self.first_month_trade_flag = 0
            self.trade_flag = 1
            return self.symbols
        else:
            return self.symbols

    def OnData(self, data):
        pass
    
    def monthly_rebalance(self):
        self.rebalence_flag = 1

    def rebalance(self):
        spy_hist = self.History([self.spy], 120, Resolution.Daily).loc[str(self.spy)]['close'] #120
        if self.Securities[self.spy].Price < spy_hist.mean():
            for symbol in self.Portfolio.Keys:
                if symbol.Value != "TLT":
                    self.Liquidate()
            self.AddEquity("TLT")
            self.SetHoldings("TLT", 1)
            return

        if self.symbols is None: return
        chosen_df = self.calc_return(self.symbols)
        chosen_df = chosen_df.iloc[:self.num_stocks]
        
        self.existing_pos = 0
        add_symbols = []
        for symbol in self.Portfolio.Keys:
            if symbol.Value == 'SPY': continue
            if (symbol.Value not in chosen_df.index):  
                self.SetHoldings(symbol, 0)
            elif (symbol.Value in chosen_df.index): 
                self.existing_pos += 1
            
        weight = 0.99/len(chosen_df)
        for symbol in chosen_df.index:
            self.AddEquity(symbol)
            self.SetHoldings(symbol, weight)    
                
    def calc_return(self, stocks):
        hist = self.History(stocks, self.formation_days, Resolution.Daily)
        current = self.History(stocks, 1, Resolution.Minute)
        
        self.price = {}
        ret = {}
     
        for symbol in stocks:
            if str(symbol) in hist.index.levels[0] and str(symbol) in current.index.levels[0]:
                self.price[symbol.Value] = list(hist.loc[str(symbol)]['close'])
                self.price[symbol.Value].append(current.loc[str(symbol)]['close'][0])
        
        for symbol in self.price.keys():
            ret[symbol] = (self.price[symbol][-1] - self.price[symbol][0]) / self.price[symbol][0]
        df_ret = pd.DataFrame.from_dict(ret, orient='index')
        df_ret.columns = ['return']
        sort_return = df_ret.sort_values(by = ['return'], ascending = self.lowmom)
        
        return sort_return

# Coarse 100 / SPY 120 mean       
# self.num_stocks = 1 100k -> 3,6 Mio., CAR 17,55%, max DD 69,4%, Sharpe 0,53, 427 Trades, 20220306 +
# self.num_stocks = 2 100k -> 5,2 Mio., CAR 19,44%, max DD 59,1%, Sharpe 0,6, 732 Trades, 20220308 +
# self.num_stocks = 3 100k -> 3,6 Mio., CAR 17,47%, max DD 56,3%, Sharpe 0,59, 1.027 Trades, 20220308 +
# 4 erl.
# 5 erl.

# Coarse 200 / SPY 120 mean       
# self.num_stocks = 1 100k -> 286k, CAR 4,85%, max DD 79,7%, Sharpe 0,28, 436 Trades, 20220306 +
# 2 erl.
# 3 erl.
# 4 erl.
# 5 erl.
# 6 erl.
# 7 erl.

# Coarse 300 / SPY 120 mean       
# self.num_stocks = 1 100k -> 4,3 Mio., CAR 18,449%, max DD 68,3%, Sharpe 0,58, 438 Trades, 20220302 +
# self.num_stocks = 2 100k -> 437k, CAR 6,885%, max DD 65,8%, Sharpe 0,34, 489 Trades, 20220302 +
# self.num_stocks = 3 100k -> 1,3 Mio, CAR 12,262%, max DD 45,6%, Sharpe 0,53, 705 Trades, 20220302 +
# self.num_stocks = 4 100k -> 786k, CAR 9,748%, max DD 51,2%, Sharpe 0,47, 924 Trades, 20220302 +
# self.num_stocks = 5 100k -> 3,7 Mio., CAR 17,681%, max DD 31,4%, Sharpe 0,71, 1.627 Trades, 20220304  +
# self.num_stocks = 6 100k -> 362k, CAR 5,969%, max DD 43,0%, Sharpe 0,34, 1.424 Trades, 20220305 ??? +
# self.num_stocks = 7 100k -> 3,1 Mio., CAR 16,817%, max DD 27,6%, Sharpe 0,72, 2.234 Trades, 20220305 +
# self.num_stocks = 8 100k -> 3,2 Mio., CAR 16,859%, max DD 27,8%, Sharpe 0,74, 2.534 Trades, 20220305 +
# self.num_stocks = 9 100k -> 3,4 Mio., CAR 17,296%, max DD 27,0%, Sharpe 0,77, 2.806 Trades, 20220305 +
# self.num_stocks = 10 100k -> 3,1 Mio., CAR 16,797%, max DD 25,5%, Sharpe 0,77, 3.109 Trades, 20220305 +
# self.num_stocks = 11 100k -> 3,0 Mio., CAR 16,621%, max DD 26,4%, Sharpe 0,77, 3.396 Trades, 20220305 +
# self.num_stocks = 12 100k -> 2,8 Mio., CAR 16,116%, max DD 25,1%, Sharpe 0,76, 3.684 Trades, 20220306 +

# Coarse 400 / SPY 120 mean       
# self.num_stocks = 1 100k -> 1,1 Mio., CAR 11,265%, max DD 52,5%, Sharpe 0,41, 438 Trades, 20220223 +
# self.num_stocks = 2 100k -> 4,4 Mio., CAR 18,646%, max DD 37,4%, Sharpe 0,65, 738 Trades, 20220224 +
# self.num_stocks = 3 100k -> 2,2 Mio., CAR 15,032%, max DD 36,3%, Sharpe 0,59, 1.046 Trades, 20220225 +
# self.num_stocks = 4 100k -> 3,2 Mio., CAR 16,819%, max DD 32,7%, Sharpe 0,67, 1.348 Trades, 20220225 +
# self.num_stocks = 5 100k -> 2,5 Mio., CAR 15,724%, max DD 33,7%, Sharpe 0,66, 1.652 Trades, 20220226 +
# self.num_stocks = 6 100k -> 3,3 Mio., CAR 17,061%, max DD 28,6%, Sharpe 0,73, 1.935 Trades, 20220226 +
# self.num_stocks = 7 100k -> 3,1 Mio., CAR 16,706%, max DD 31,4%, Sharpe 0,73, 2.228 Trades, 20220226 +
# self.num_stocks = 8 100k -> 2,9 Mio., CAR 16,392%, max DD 31,2%, Sharpe 0,73, 2.525 Trades, 20220227 + 
# self.num_stocks = 9 100k -> 2,7 Mio., CAR 16,017%, max DD 28,5%, Sharpe 0,73, 2.817 Trades, 20220227 +
# self.num_stocks = 10 100k -> 2,8 Mio., CAR 16,282%, max DD 28,1%, Sharpe 0,75, 3.095 Trades, 20220227 +
# self.num_stocks = 11 100k -> 2,7 Mio., CAR 16,012%, max DD 27,0%, Sharpe 0,75, 3.376 Trades, 20220227 +
# self.num_stocks = 12 100k -> 3,0 Mio., CAR 16,498%, max DD 26,0%, Sharpe 0,78, 3.643 Trades, 20220227 +
# self.num_stocks = 13 100k -> 2,7 Mio., CAR 16,104%, max DD 25,7%, Sharpe 0,78, 3.933 Trades, 20220228 +
# self.num_stocks = 14 100k -> 2,5 Mio., CAR 15,597%, max DD 25,2%, Sharpe 0,76, 4.210 Trades, 20220228 +
# self.num_stocks = 15 100k -> 3,0 Mio., CAR 16,629%, max DD 24,4%, Sharpe 0,81, 4.486 Trades, 20220228 +

# Coarse 500 / SPY 120 mean       

# self.num_stocks = 1 100k -> 3,0 Mio., CAR 16,567%, max DD 58,4%, Sharpe 0,54, 426 Trades +
# self.num_stocks = 2 100k -> 5,3 Mio., CAR 19,65%%, max DD 47,9%, Sharpe 0,67, 745 Trades +
# self.num_stocks = 3 100k -> 7,4 Mio., CAR 21,534%, max DD 43,9%, Sharpe 0,78, 1.046 Trades +
# self.num_stocks = 4 100k -> 8,7 Mio., CAR 22,435%, max DD 33,0%, Sharpe 0,84, 1.349 Trades +
# self.num_stocks = 5 100k -> 8,6 Mio., CAR 22,345%, max DD 27,1%, Sharpe 0,88, 1.635 Trades+
# self.num_stocks = 6 100k -> 7,3 Mio., CAR 21,466%, max DD 24,6%, Sharpe 0,87, 1.925 Trades +
# self.num_stocks = 7 100k -> 6,3 Mio., CAR 20,654%, max DD 24,5%, Sharpe 0,86, 2.218 Trades +
# self.num_stocks = 8 100k -> 4,8 Mio., CAR 19,168%, max DD 24,7%, Sharpe 0,83, 2.507 Trades +
# self.num_stocks = 9 100k -> 4,3 Mio., CAR 18,505%, max DD 27,8%, Sharpe 0,81, 2.764 Trades+
# self.num_stocks = 10 100k -> 4,0 Mio., CAR 18,143%, max DD 23,2% %, Sharpe 0,82, 3.039 Trades +
# self.num_stocks = 11 100k -> 4,1 Mio., CAR 18,273%, max DD 26,3% %, Sharpe 0,84, 3.343 Trades +
# self.num_stocks = 12 100k -> 3,9 Mio., CAR 17,958%, max DD 27,7% %, Sharpe 0,84, 3.626 Trades +
# self.num_stocks = 13 100k -> 3,6 Mio., CAR 17,586%, max DD 27,3% %, Sharpe 0,83, 3.918 Trades +

# self.num_stocks = 15 100k -> 3,7 Mio., CAR 17,772%, max DD 27,1% %, Sharpe 0,69, 3.334 Trades +

# Coarse 600 / SPY 120 mean       
# self.num_stocks = 1 100k --> 1,0 Mio., CAR 10,983%, max DD 65,6%, Sharpe 0,41, 420 Trades, 20220206 +
# self.num_stocks = 2 100k --> 1,6 Mio., CAR 10,983%, max DD 65,6%, Sharpe 0,41, 420 Trades, 20220206 +
# self.num_stocks = 3 100k --> 2,0 Mio., CAR 13,218%, max DD 57,6%, Sharpe 0,5, 750 Trades, 20220207 +
# self.num_stocks = 4 100k --> 3,5 Mio., CAR 17,519%, max DD 34,9%, Sharpe 0,69, 1.359 Trades, 20220206 +
# self.num_stocks = 5 100k --> 5,0 Mio., CAR 19,308%, max DD 28,4%, Sharpe 0,77, 1.646 Trades, 20220209 +
# self.num_stocks = 6 100k --> 4,8 Mio., CAR 19,118%, max DD 29,5%, Sharpe 0,79, 1.946 Trades, 20220209
# self.num_stocks = 7 100k --> 4,6 Mio., CAR 18,904%, max DD 26,8%, Sharpe 0,8, 2.220 Trades, 20220209
# self.num_stocks = 8 100k --> 4,0 Mio., CAR 18,103%, max DD 24,5%, Sharpe 0,79, 2.514 Trades, 20220211
# self.num_stocks = 9 100k --> 3,4 Mio., CAR 17,285%, max DD 24,6%, Sharpe 0,77, 2.798 Trades, 20220211
# self.num_stocks = 10 100k --> 3,1 Mio., CAR 16,858%, max DD 24,6%, Sharpe 0,76, 3.075 Trades, 20220217

# self.num_stocks = 15 100k --> 2,4 Mio., CAR 15,377%, max DD 28,6%, Sharpe 0,75, 4.484 Trades, 20220211


# Coarse 700 / SPY 120 mean       
# self.num_stocks = 1 100k --> 1,0 Mio., CAR 11,139%, max DD 61,1%, Sharpe 0,41, 437 Trades, 20220306 +
# self.num_stocks = 5 100k --> 3,1 Mio., CAR 16,829%, max DD 28,6%, Sharpe 0,69, 1.658 Trades, 20220206 +

# Coarse 800 / SPY 120 mean       
# self.num_stocks = 1 100k --> 1,4 Mio., CAR 12,5%%, max DD 57,1%%, Sharpe 0,45, 444 Trades, 20220306 +
# self.num_stocks = 2 100k --> 2,1 Mio., CAR 14,6%, max DD 51,1%%, Sharpe 0,53, 763 Trades, 20220306 +
# self.num_stocks = 3 100k --> 1,7 Mio., CAR 13,75%, max DD 35,4%, Sharpe 0,54, 1.082 Trades, 20220306 +

# self.num_stocks = 5 100k --> 3,2 Mio., CAR 16,906%, max DD 30,0%, Sharpe 0,69, 1.654 Trades, 20220206 +
# self.num_stocks = 6 100k --> 3,8 Mio., CAR 17,896%, max DD 26,4%, Sharpe 0,75, 1.933 Trades, 20220207 +
# self.num_stocks = 7 100k --> 3,8 Mio., CAR 17,938%, max DD 25,6%, Sharpe 0,77, 2.213 Trades, 20220207 +
# self.num_stocks = 8 100k --> 3,3 Mio., CAR 17,109%, max DD 28,4%, Sharpe 0,75, 2.510 Trades, 20220207 +

# Coarse 900 / SPY 120 mean       
# self.num_stocks = 1 100k --> 2,2 Mio., CAR 14,958%, max DD 57,1%, Sharpe 0,5, 437 Trades, 20220306 +
# self.num_stocks = 8 100k --> 3,2 Mio., CAR 17,041%, max DD 31,1%, Sharpe 0,75, 2.500Trades, 20220207 +
# 10 erl.
# 11 erl.
# 12 erl.
# 13 erl.
# 14 erl.
# 15 erl.

# Coarse 1000 / SPY 120 mean       
# self.num_stocks = 1 100k --> 3,2 Mio., CAR 18,87%, max DD 57,1%, Sharpe 0,54, 439 Trades, 20220306 +
# self.num_stocks = 13 100k --> 2,5 Mio., CAR 15,52%, max DD 29,5%, Sharpe 0,74, 3.942 Trades, 20220307 +
# self.num_stocks = 14 100k --> 2,2 Mio., CAR 15,04%, max DD 28,6%, Sharpe 0,73, 4.239 Trades, 20220307 +
# self.num_stocks = 15 100k --> 2,1 Mio., CAR 14,69%, max DD 30,0%, Sharpe 0,72, 4.521 Trades, 20220307 +

# Coarse 500 / SPY 200 mean       
# self.num_stocks = 1 100k -> 6,1 Mio., CAR 20,418%, max DD 53,2% %, Sharpe 0,62, 416 Trades +
# self.num_stocks = 2 100k -> 6,8 Mio., CAR 20,996%, max DD 45,4% %, Sharpe 0,7, 737 Trade+s
# self.num_stocks = 3 100k -> 8,7 Mio., CAR 22,406%, max DD 45,6% %, Sharpe 0,79, 1.044 Tr+ades
# self.num_stocks = 4 100k -> 6,4 Mio., CAR 20,680%, max DD 40,9% %, Sharpe 0,78, 1.357 Trades +
# self.num_stocks = 5 100k -> 7,1 Mio., CAR 21,251%, max DD 36,8% %, Sharpe 0,82, 1.660 Trades +
# self.num_stocks = 6 100k -> 5,9 Mio., CAR 20,263%, max DD 32,9% %, Sharpe 0,80, 1.951 Trades+
# self.num_stocks = 7 100k -> 5,4 Mio., CAR 19,760%, max DD 33,1% %, Sharpe 0,81, 2.250 Trades+
# self.num_stocks = 8 100k -> 4,2 Mio., CAR 18,426%, max DD 34,7% %, Sharpe 0,77, 2.542 Trades, 20220205+


# ohne ADR / SPY 120 mean
# self.num_stocks = 10 100k -> 2,6 Mio., CAR 15,871%, max DD 27,3% %, Sharpe 0,7, 3.024 Trades