# https://quantpedia.com/strategies/value-factor-effect-within-countries/
#
# The investment universe consists of 32 countries with easily accessible equity markets (via ETFs, for example). At the end of every year, 
# the investor calculates Shiller’s “CAPE” Cyclically Adjusted PE) ratio, for each country in his investment universe. CAPE is the ratio of 
# the real price of the equity market (adjusted for inflation) to the 10-year average of the country’s equity index (again adjusted for inflation). 
# The whole methodology is explained well on Shiller’s home page (http://www.econ.yale.edu/~shiller/data.htm) or
# http://turnkeyanalyst.com/2011/10/the-shiller-pe-ratio/). The investor then invests in the cheapest 33% of countries from his sample if those 
# countries have a CAPE below 15. The portfolio is equally weighted (the investor holds 0% cash instead of countries with a CAPE higher than 15)
# and rebalanced yearly.

from fk_tools import CustomFeeModel

class ValueFactorCAPEEffectwithinCountries(QCAlgorithm):

    def Initialize(self):
        self.SetStartDate(2008, 1, 1)  
        self.SetCash(100000)

        self.symbols = {
            "Australia"     : "EWA",  # iShares MSCI Australia Index ETF
            "Brazil"        : "EWZ",  # iShares MSCI Brazil Index ETF
            "Canada"        : "EWC",  # iShares MSCI Canada Index ETF
            "Switzerland"   : "EWL",  # iShares MSCI Switzerland Index ETF
            "China"         : "FXI",  # iShares China Large-Cap ETF
            "France"        : "EWQ",  # iShares MSCI France Index ETF
            "Germany"       : "EWG",  # iShares MSCI Germany ETF 
            "Hong Kong"     : "EWH",  # iShares MSCI Hong Kong Index ETF
            "Italy"         : "EWI",  # iShares MSCI Italy Index ETF
            "Japan"         : "EWJ",  # iShares MSCI Japan Index ETF
            "Korea"         : "EWY",  # iShares MSCI South Korea ETF
            "Mexico"        : "EWW",  # iShares MSCI Mexico Inv. Mt. Idx
            "Netherlands"   : "EWN",  # iShares MSCI Netherlands Index ETF
            "South Africa"  : "EZA",  # iShares MSCI South Africe Index ETF
            "Singapore"     : "EWS",  # iShares MSCI Singapore Index ETF
            "Spain"         : "EWP",  # iShares MSCI Spain Index ETF
            "Sweden"        : "EWD",  # iShares MSCI Sweden Index ETF
            "Taiwan"        : "EWT",  # iShares MSCI Taiwan Index ETF
            "UK"            : "EWU",  # iShares MSCI United Kingdom Index ETF
            "USA"           : "SPY",  # SPDR S&P 500 ETF
            
            "Russia"        : "ERUS",  # iShares MSCI Russia ETF
            "Israel"        : "EIS",   # iShares MSCI Israel ETF
            "India"         : "INDA",  # iShares MSCI India ETF
            "Poland"        : "EPOL",  # iShares MSCI Poland ETF
            "Turkey"        : "TUR"    # iShares MSCI Turkey ETF
        }

        for country, etf_symbol in self.symbols.items():
            data = self.AddEquity(etf_symbol, Resolution.Daily)
            data.SetFeeModel(CustomFeeModel(self))
            
        # CAPE data import.
        self.AddData(CAPE, 'CAPE',  Resolution.Daily)
            
        self.month = 11
        
        self.Schedule.On(self.DateRules.MonthStart('EWA'), self.TimeRules.AfterMarketOpen('EWA'), self.Rebalance)
        
    def Rebalance(self):
        self.month += 1
        if self.month > 12:
            self.month = 1
            
        if self.month != 12:
            return
        
        price = {}
        for country, etf_symbol in self.symbols.items():
            if self.Securities.ContainsKey('CAPE') and self.Securities.ContainsKey(etf_symbol):
                cape_data = self.Securities['CAPE'].GetLastData()
                if cape_data:
                    country_cape = cape_data.GetProperty(country)
                    if country_cape < 15:
                        etf_price = self.Securities[etf_symbol].Price
                        if etf_price != 0:
                            price[etf_symbol] = etf_price
        
        long = []
        
        # Cape and price sorting.
        if len(price) != 0:
            sorted_by_price = sorted(price.items(), key = lambda x: x[1], reverse = True)
            tercile = int(len(sorted_by_price) / 3)
            long = [x[0] for x in sorted_by_price[-tercile:]]
        
        # Trade execution.
        invested = [x.Key for x in self.Portfolio if x.Value.Invested]
        for symbol in invested:
            if symbol not in long:
                self.Liquidate(symbol)
        
        for symbol in long:
            if self.Securities[etf_symbol].Price != 0 and self.Securities[etf_symbol].IsTradable:
                self.SetHoldings(symbol, 1 / len(long))

# NOTE: IMPORTANT: Data order must be ascending (datewise)
# Data source: https://indices.barclays/IM/21/en/indices/static/historic-cape.app
class CAPE(PythonData):
    def GetSource(self, config, date, isLiveMode):
        return SubscriptionDataSource("data.quantpedia.com/backtesting_data/economic/cape_by_country.csv", SubscriptionTransportMedium.RemoteFile, FileFormat.Csv)

    def Reader(self, config, line, date, isLiveMode):
        data = CAPE()
        data.Symbol = config.Symbol
        
        if not line[0].isdigit(): return None
        split = line.split(';')
        
        data.Time = datetime.strptime(split[0], "%Y-%m-%d") + timedelta(days=1)
        
        data['Australia'] = float(split[1])
        data['Brazil'] = float(split[2])
        data['Canada'] = float(split[3])
        data['Switzerland'] = float(split[4])
        data['China'] = float(split[5])
        data['France'] = float(split[6])
        data['Germany'] = float(split[7])
        data['Hong Kong'] = float(split[8])
        data['India'] = float(split[9])
        data['Israel'] = float(split[10])
        data['Italy'] = float(split[11])
        data['Japan'] = float(split[12])
        data['Korea'] = float(split[13])
        data['Mexico'] = float(split[14])
        data['Netherlands'] = float(split[15])
        data['Poland'] = float(split[16])
        data['Russia'] = float(split[17])
        data['South Africa'] = float(split[18])
        data['Singapore'] = float(split[19])
        data['Spain'] = float(split[20])
        data['Sweden'] = float(split[21])
        data['Taiwan'] = float(split[22])
        data['Turkey'] = float(split[23])
        data['UK'] = float(split[24])
        data['USA'] = float(split[25])
        
        data.Value = float(split[1])

        return data

import numpy as np
from scipy.optimize import minimize

sp100_stocks = ['AAPL','MSFT','AMZN','FB','BRKB','GOOGL','GOOG','JPM','JNJ','V','PG','XOM','UNH','BAC','MA','T','DIS','INTC','HD','VZ','MRK','PFE','CVX','KO','CMCSA','CSCO','PEP','WFC','C','BA','ADBE','WMT','CRM','MCD','MDT','BMY','ABT','NVDA','NFLX','AMGN','PM','PYPL','TMO','COST','ABBV','ACN','HON','NKE','UNP','UTX','NEE','IBM','TXN','AVGO','LLY','ORCL','LIN','SBUX','AMT','LMT','GE','MMM','DHR','QCOM','CVS','MO','LOW','FIS','AXP','BKNG','UPS','GILD','CHTR','CAT','MDLZ','GS','USB','CI','ANTM','BDX','TJX','ADP','TFC','CME','SPGI','COP','INTU','ISRG','CB','SO','D','FISV','PNC','DUK','SYK','ZTS','MS','RTN','AGN','BLK']

def MonthDiff(d1, d2):
    return (d1.year - d2.year) * 12 + d1.month - d2.month

def Return(values):
    return (values[-1] - values[0]) / values[0]
    
def Volatility(values):
    values = np.array(values)
    returns = (values[1:] - values[:-1]) / values[:-1]
    return np.std(returns)  

def GetFutureMulitpliers(algorithm):
    symbol_multiplier = {}
    
    csv_string_file = algorithm.Download('data.quantpedia.com/backtesting_data/futures/contract_multiplier.csv')
    mulitpliers_lines = csv_string_file.split('\r\n')
    for line in mulitpliers_lines:
        symbol, multiplier = line.split(';')
        symbol_multiplier[symbol] = multiplier
    
    return symbol_multiplier

# Custom fee model
class CustomFeeModel(FeeModel):
    def GetOrderFee(self, parameters):
        fee = parameters.Security.Price * parameters.Order.AbsoluteQuantity * 0.00005
        return OrderFee(CashAmount(fee, "USD"))

# Quandl free data
class QuandlFutures(PythonQuandl):
    def __init__(self):
        self.ValueColumnName = "settle"

# Quandl short interest data.
class QuandlFINRA_ShortVolume(PythonQuandl):
    def __init__(self):
        self.ValueColumnName = 'SHORTVOLUME'    # also 'TOTALVOLUME' is accesible

# Quantpedia data.
# NOTE: IMPORTANT: Data order must be ascending (datewise)
class QuantpediaFutures(PythonData):
    def GetSource(self, config, date, isLiveMode):
        return SubscriptionDataSource("data.quantpedia.com/backtesting_data/futures/{0}.csv".format(config.Symbol.Value), SubscriptionTransportMedium.RemoteFile, FileFormat.Csv)

    def Reader(self, config, line, date, isLiveMode):
        data = QuantpediaFutures()
        data.Symbol = config.Symbol
        
        if not line[0].isdigit(): return None
        split = line.split(';')
        
        data.Time = datetime.strptime(split[0], "%d.%m.%Y") + timedelta(days=1)
        data['back_adjusted'] = float(split[1])
        data['spliced'] = float(split[2])
        data.Value = float(split[1])

        return data

# Commitments of Traders data.
# NOTE: IMPORTANT: Data order must be ascending (datewise).
# Data source: https://commitmentsoftraders.org/cot-data/
# Data description: https://commitmentsoftraders.org/wp-content/uploads/Static/CoTData/file_key.html
class CommitmentsOfTraders(PythonData):
    def GetSource(self, config, date, isLiveMode):
        return SubscriptionDataSource("data.quantpedia.com/backtesting_data/futures/cot/{0}.PRN".format(config.Symbol.Value), SubscriptionTransportMedium.RemoteFile, FileFormat.Csv)

    # File example.
    # DATE   OPEN     HIGH        LOW       CLOSE     VOLUME   OI
    # ----   ----     ----        ---       -----     ------   --
    # DATE   LARGE    SPECULATOR  COMMERCIAL HEDGER   SMALL TRADER
    #        LONG     SHORT       LONG      SHORT     LONG     SHORT
    def Reader(self, config, line, date, isLiveMode):
        data = CommitmentsOfTraders()
        data.Symbol = config.Symbol
        
        if not line[0].isdigit(): return None
        split = line.split(',')
        
        # Prevent lookahead bias.
        data.Time = datetime.strptime(split[0], "%Y%m%d") + timedelta(days=1)
        
        data['LARGE_SPECULATOR_LONG'] = int(split[1])
        data['LARGE_SPECULATOR_SHORT'] = int(split[2])
        data['COMMERCIAL_HEDGER_LONG'] = int(split[3])
        data['COMMERCIAL_HEDGER_SHORT'] = int(split[4])
        data['SMALL_TRADER_LONG'] = int(split[5])
        data['SMALL_TRADER_SHORT'] = int(split[6])

        data.Value = int(split[1])

        return data
        
# NOTE: Manager for new trades. It's represented by certain count of equally weighted brackets for long and short positions.
# If there's a place for new trade, it will be managed for time of holding period.
class TradeManager():
    def __init__(self, algorithm, long_size, short_size, holding_period):
        self.algorithm = algorithm  # algorithm to execute orders in.
        
        self.long_size = long_size
        self.short_size = short_size
        
        self.long_len = 0
        self.short_len = 0
    
        # Arrays of ManagedSymbols
        self.symbols = []
        
        self.holding_period = holding_period    # Days of holding.
    
    # Add stock symbol object
    def Add(self, symbol, long_flag):
        # Open new long trade.
        managed_symbol = ManagedSymbol(symbol, self.holding_period, long_flag)
        
        if long_flag:
            # If there's a place for it.
            if self.long_len < self.long_size:
                self.symbols.append(managed_symbol)
                self.algorithm.SetHoldings(symbol, 1 / self.long_size)
                self.long_len += 1
            else:
                self.algorithm.Log("There's not place for additional trade.")

        # Open new short trade.
        else:
            # If there's a place for it.
            if self.short_len < self.short_size:
                self.symbols.append(managed_symbol)
                self.algorithm.SetHoldings(symbol, - 1 / self.short_size)
                self.short_len += 1
            else:
                self.algorithm.Log("There's not place for additional trade.")
   
    # Decrement holding period and liquidate symbols.
    def TryLiquidate(self):
        symbols_to_delete = []
        for managed_symbol in self.symbols:
            managed_symbol.days_to_liquidate -= 1
            
            # Liquidate.
            if managed_symbol.days_to_liquidate == 0:
                symbols_to_delete.append(managed_symbol)
                self.algorithm.Liquidate(managed_symbol.symbol)
                
                if managed_symbol.long_flag: self.long_len -= 1
                else: self.short_len -= 1

        # Remove symbols from management.
        for managed_symbol in symbols_to_delete:
            self.symbols.remove(managed_symbol)
    
    def LiquidateTicker(self, ticker):
        symbol_to_delete = None
        for managed_symbol in self.symbols:
            if managed_symbol.symbol.Value == ticker:
                self.algorithm.Liquidate(managed_symbol.symbol)
                symbol_to_delete = managed_symbol
                if managed_symbol.long_flag: self.long_len -= 1
                else: self.short_len -= 1
                
                break
        
        if symbol_to_delete: self.symbols.remove(symbol_to_delete)
        else: self.algorithm.Debug("Ticker is not held in portfolio!")
    
class ManagedSymbol():
    def __init__(self, symbol, days_to_liquidate, long_flag):
        self.symbol = symbol
        self.days_to_liquidate = days_to_liquidate
        self.long_flag = long_flag
        
class PortfolioOptimization(object):
    def __init__(self, df_return, risk_free_rate, num_assets):
        self.daily_return = df_return
        self.risk_free_rate = risk_free_rate
        self.n = num_assets # numbers of risk assets in portfolio
        self.target_vol = 0.05

    def annual_port_return(self, weights):
        # calculate the annual return of portfolio
        return np.sum(self.daily_return.mean() * weights) * 252

    def annual_port_vol(self, weights):
        # calculate the annual volatility of portfolio
        return np.sqrt(np.dot(weights.T, np.dot(self.daily_return.cov() * 252, weights)))

    def min_func(self, weights):
        # method 1: maximize sharp ratio
        return - self.annual_port_return(weights) / self.annual_port_vol(weights)
        
        # method 2: maximize the return with target volatility
        #return - self.annual_port_return(weights) / self.target_vol

    def opt_portfolio(self):
        # maximize the sharpe ratio to find the optimal weights
        cons = ({'type': 'eq', 'fun': lambda x: np.sum(x) - 1})
        bnds = tuple((0, 1) for x in range(2)) + tuple((0, 0.25) for x in range(self.n - 2))
        opt = minimize(self.min_func,                               # object function
                       np.array(self.n * [1. / self.n]),            # initial value
                       method='SLSQP',                              # optimization method
                       bounds=bnds,                                 # bounds for variables 
                       constraints=cons)                            # constraint conditions
                      
        opt_weights = opt['x']
 
        return opt_weights