import numpy as np
from scipy.optimize import minimize

sp100_stocks = ['AAPL','MSFT','AMZN','FB','BRKB','GOOGL','GOOG','JPM','JNJ','V','PG','XOM','UNH','BAC','MA','T','DIS','INTC','HD','VZ','MRK','PFE','CVX','KO','CMCSA','CSCO','PEP','WFC','C','BA','ADBE','WMT','CRM','MCD','MDT','BMY','ABT','NVDA','NFLX','AMGN','PM','PYPL','TMO','COST','ABBV','ACN','HON','NKE','UNP','UTX','NEE','IBM','TXN','AVGO','LLY','ORCL','LIN','SBUX','AMT','LMT','GE','MMM','DHR','QCOM','CVS','MO','LOW','FIS','AXP','BKNG','UPS','GILD','CHTR','CAT','MDLZ','GS','USB','CI','ANTM','BDX','TJX','ADP','TFC','CME','SPGI','COP','INTU','ISRG','CB','SO','D','FISV','PNC','DUK','SYK','ZTS','MS','RTN','AGN','BLK']

def MonthDiff(d1, d2):
    return (d1.year - d2.year) * 12 + d1.month - d2.month

def Return(values):
    return (values[-1] - values[0]) / values[0]
    
def Volatility(values):
    values = np.array(values)
    returns = (values[1:] - values[:-1]) / values[:-1]
    return np.std(returns)

def GetFutureMulitpliers(algorithm):
    symbol_multiplier = {}
    
    csv_string_file = algorithm.Download('data.quantpedia.com/backtesting_data/futures/contract_multiplier.csv')
    mulitpliers_lines = csv_string_file.split('\r\n')
    for line in mulitpliers_lines:
        symbol, multiplier = line.split(';')
        symbol_multiplier[symbol] = float(multiplier)
    
    return symbol_multiplier

# Custom fee model
class CustomFeeModel(FeeModel):
    def GetOrderFee(self, parameters):
        fee = parameters.Security.Price * parameters.Order.AbsoluteQuantity * 0.00005
        return OrderFee(CashAmount(fee, "USD"))

# Quandl free data.
class QuandlFutures(PythonQuandl):
    def __init__(self):
        self.ValueColumnName = "settle"

# Quandl short interest data.
class QuandlFINRA_ShortVolume(PythonQuandl):
    def __init__(self):
        self.ValueColumnName = 'SHORTVOLUME'    # also 'TOTALVOLUME' is accesible

# Quantpedia data.
# NOTE: IMPORTANT: Data order must be ascending (datewise)
class QuantpediaFutures(PythonData):
    def GetSource(self, config, date, isLiveMode):
        return SubscriptionDataSource("data.quantpedia.com/backtesting_data/futures/{0}.csv".format(config.Symbol.Value), SubscriptionTransportMedium.RemoteFile, FileFormat.Csv)

    def Reader(self, config, line, date, isLiveMode):
        data = QuantpediaFutures()
        data.Symbol = config.Symbol
        
        if not line[0].isdigit(): return None
        split = line.split(';')
        
        data.Time = datetime.strptime(split[0], "%d.%m.%Y") + timedelta(days=1)
        data['back_adjusted'] = float(split[1])
        data['spliced'] = float(split[2])
        data.Value = float(split[1])

        return data

# NOTE: Manager for new trades. It's represented by certain count of equally weighted brackets for long and short positions.
# If there's a place for new trade, it will be managed for time of holding period.
class TradeManager():
    def __init__(self, algorithm, long_size, short_size, holding_period):
        self.algorithm = algorithm  # algorithm to execute orders in.
        
        self.long_size = long_size
        self.short_size = short_size
        self.weight = 1 / (self.long_size + self.short_size)
        
        self.long_len = 0
        self.short_len = 0
    
        # Arrays of ManagedSymbols
        self.symbols = []
        
        self.holding_period = holding_period    # Days of holding.
    
    # Add stock symbol object
    def Add(self, symbol, long_flag):
        # Open new long trade.
        managed_symbol = ManagedSymbol(symbol, self.holding_period, long_flag)
        
        if long_flag:
            # If there's a place for it.
            if self.long_len < self.long_size:
                self.symbols.append(managed_symbol)
                self.algorithm.SetHoldings(symbol, self.weight)
                self.long_len += 1
            else:
                self.algorithm.Log("There's not place for additional trade.")

        # Open new short trade.
        else:
            # If there's a place for it.
            if self.short_len < self.short_size:
                self.symbols.append(managed_symbol)
                self.algorithm.SetHoldings(symbol, - self.weight)
                self.short_len += 1
            else:
                self.algorithm.Log("There's not place for additional trade.")
    
    # Decrement holding period and liquidate symbols.
    def TryLiquidate(self):
        symbols_to_delete = []
        for managed_symbol in self.symbols:
            managed_symbol.days_to_liquidate -= 1
            
            # Liquidate.
            if managed_symbol.days_to_liquidate == 0:
                symbols_to_delete.append(managed_symbol)
                self.algorithm.Liquidate(managed_symbol.symbol)
                
                if managed_symbol.long_flag: self.long_len -= 1
                else: self.short_len -= 1

        # Remove symbols from management.
        for managed_symbol in symbols_to_delete:
            self.symbols.remove(managed_symbol)
    
    def LiquidateTicker(self, ticker):
        symbol_to_delete = None
        for managed_symbol in self.symbols:
            if managed_symbol.symbol.Value == ticker:
                self.algorithm.Liquidate(managed_symbol.symbol)
                symbol_to_delete = managed_symbol
                if managed_symbol.long_flag: self.long_len -= 1
                else: self.short_len -= 1
                
                break
        
        if symbol_to_delete: self.symbols.remove(symbol_to_delete)
        else: self.algorithm.Debug("Ticker is not held in portfolio!")
    
class ManagedSymbol():
    def __init__(self, symbol, days_to_liquidate, long_flag):
        self.symbol = symbol
        self.days_to_liquidate = days_to_liquidate
        self.long_flag = long_flag
        
class PortfolioOptimization(object):
    def __init__(self, df_return, risk_free_rate, num_assets):
        self.daily_return = df_return
        self.risk_free_rate = risk_free_rate
        self.n = num_assets # numbers of risk assets in portfolio
        self.target_vol = 0.05

    def annual_port_return(self, weights):
        # calculate the annual return of portfolio
        return np.sum(self.daily_return.mean() * weights) * 252

    def annual_port_vol(self, weights):
        # calculate the annual volatility of portfolio
        return np.sqrt(np.dot(weights.T, np.dot(self.daily_return.cov() * 252, weights)))

    def min_func(self, weights):
        # method 1: maximize sharp ratio
        return - self.annual_port_return(weights) / self.annual_port_vol(weights)
        
        # method 2: maximize the return with target volatility
        #return - self.annual_port_return(weights) / self.target_vol

    def opt_portfolio(self):
        # maximize the sharpe ratio to find the optimal weights
        cons = ({'type': 'eq', 'fun': lambda x: np.sum(x) - 1})
        bnds = tuple((0, 1) for x in range(2)) + tuple((0, 0.25) for x in range(self.n - 2))
        opt = minimize(self.min_func,                               # object function
                       np.array(self.n * [1. / self.n]),            # initial value
                       method='SLSQP',                              # optimization method
                       bounds=bnds,                                 # bounds for variables 
                       constraints=cons)                            # constraint conditions
                      
        opt_weights = opt['x']
 
        return opt_weights

# https://quantpedia.com/strategies/time-series-momentum-effect/
#
# The investment universe consists of 24 commodity futures, 12 cross-currency pairs (with 9 underlying currencies), 9 developed equity indices, and 13 developed
# government bond futures.
# Every month, the investor considers whether the excess return of each asset over the past 12 months is positive or negative and goes long on the contract if it is 
# positive and short if negative. The position size is set to be inversely proportional to the instrument’s volatility. A univariate GARCH model is used to estimated 
# ex-ante volatility in the source paper. However, other simple models could probably be easily used with good results (for example, the easiest one would be using 
# historical volatility instead of estimated volatility). The portfolio is rebalanced monthly.

from collections import deque
import fk_tools
from math import sqrt

class TimeSeriesMomentum(QCAlgorithm):

    def Initialize(self):
        self.SetStartDate(2000, 1, 1)
        self.SetCash(100000)

        self.symbols = [
                        "CME_S1",   # Soybean Futures, Continuous Contract
                        "CME_W1",   # Wheat Futures, Continuous Contract
                        "CME_SM1",  # Soybean Meal Futures, Continuous Contract
                        "CME_BO1",  # Soybean Oil Futures, Continuous Contract
                        "CME_C1",   # Corn Futures, Continuous Contract
                        "CME_O1",   # Oats Futures, Continuous Contract
                        "CME_LC1",  # Live Cattle Futures, Continuous Contract
                        "CME_FC1",  # Feeder Cattle Futures, Continuous Contract
                        "CME_LN1",  # Lean Hog Futures, Continuous Contract
                        "CME_GC1",  # Gold Futures, Continuous Contract
                        "CME_SI1",  # Silver Futures, Continuous Contract
                        "CME_PL1",  # Platinum Futures, Continuous Contract
                        "CME_CL1",  # Crude Oil Futures, Continuous Contract
                        "CME_HG1",  # Copper Futures, Continuous Contract
                        "CME_LB1",  # Random Length Lumber Futures, Continuous Contract
                        "CME_NG1",  # Natural Gas (Henry Hub) Physical Futures, Continuous Contract
                        "CME_PA1",  # Palladium Futures, Continuous Contract 
                        "CME_RR1",  # Rough Rice Futures, Continuous Contract
                        "CME_CU1",  # Chicago Ethanol (Platts) Futures
                        "CME_DA1",  # Class III Milk Futures
                        
                        "ICE_CC1",  # Cocoa Futures, Continuous Contract 
                        "ICE_CT1",  # Cotton No. 2 Futures, Continuous Contract
                        "ICE_KC1",  # Coffee C Futures, Continuous Contract
                        "ICE_O1",   # Heating Oil Futures, Continuous Contract
                        "ICE_OJ1",  # Orange Juice Futures, Continuous Contract
                        "ICE_SB1",  # Sugar No. 11 Futures, Continuous Contract
                                                
                        "CME_AD1", # Australian Dollar Futures, Continuous Contract #1
                        "CME_BP1", # British Pound Futures, Continuous Contract #1
                        "CME_CD1", # Canadian Dollar Futures, Continuous Contract #1
                        "CME_EC1", # Euro FX Futures, Continuous Contract #1
                        "CME_JY1", # Japanese Yen Futures, Continuous Contract #1
                        "CME_MP1", # Mexican Peso Futures, Continuous Contract #1
                        "CME_NE1", # New Zealand Dollar Futures, Continuous Contract #1
                        "CME_SF1", # Swiss Franc Futures, Continuous Contract #1
                    
                        "ICE_DX1",      # US Dollar Index Futures, Continuous Contract #1
                        "CME_NQ1",      # E-mini NASDAQ 100 Futures, Continuous Contract #1
                        "EUREX_FDAX1",  # DAX Futures, Continuous Contract #1
                        "CME_ES1",      # E-mini S&P 500 Futures, Continuous Contract #1
                        "EUREX_FSMI1",  # SMI Futures, Continuous Contract #1
                        "EUREX_FSTX1",  # STOXX Europe 50 Index Futures, Continuous Contract #1
                        "LIFFE_FCE1",   # CAC40 Index Futures, Continuous Contract #1
                        "LIFFE_Z1",     # FTSE 100 Index Futures, Continuous Contract #1
                        "SGX_NK1",      # SGX Nikkei 225 Index Futures, Continuous Contract #1
                        "CME_MD1",      # E-mini S&P MidCap 400 Futures
                        
                        "CME_TY1",      # 10 Yr Note Futures, Continuous Contract #1
                        "CME_FV1",      # 5 Yr Note Futures, Continuous Contract #1
                        "CME_TU1",      # 2 Yr Note Futures, Continuous Contract #1
                        "ASX_XT1",     # 10 Year Commonwealth Treasury Bond Futures, Continuous Contract #1   # 'Settlement price' instead of 'settle' on quandl. 
                        "ASX_YT1",     # 3 Year Commonwealth Treasury Bond Futures, Continuous Contract #1    # 'Settlement price' instead of 'settle' on quandl.
                        "EUREX_FGBL1",  # Euro-Bund (10Y) Futures, Continuous Contract #1
                        "EUREX_FBTP1", # Long-Term Euro-BTP Futures, Continuous Contract #1
                        "EUREX_FGBM1",  # Euro-Bobl Futures, Continuous Contract #1
                        "EUREX_FGBS1",  # Euro-Schatz Futures, Continuous Contract #1 
                        "SGX_JB1",      # SGX 10-Year Mini Japanese Government Bond Futures
                        "LIFFE_R1"      # Long Gilt Futures, Continuous Contract #1
                        "MX_CGB1",     # Ten-Year Government of Canada Bond Futures, Continuous Contract #1    # 'Settlement price' instead of 'settle' on quandl.
                    ]
                    
        self.period = 12 * 21
        self.SetWarmUp(self.period)
        
        self.targeted_volatility = 0.10
        
        # Daily rolled data.
        self.data = {}
        
        for symbol in self.symbols:
            data = None
            
            # Back adjusted and spliced data import.
            data = self.AddData(fk_tools.QuantpediaFutures, symbol, Resolution.Daily)
            
            data.SetFeeModel(fk_tools.CustomFeeModel(self))
            data.SetLeverage(20)
            
            self.data[symbol] = deque(maxlen=self.period)
        
        self.Schedule.On(self.DateRules.MonthStart(self.symbols[0]), self.TimeRules.AfterMarketOpen(self.symbols[0]), self.Rebalance)
    
    def OnData(self, data):
        # Store daily data.
        for symbol in self.symbols:
            if self.Securities.ContainsKey(symbol):
                price = self.Securities[symbol].Price
                if price != 0:
                    self.data[symbol].append(price)
                else:
                    # Append latest price as a next one in case there's 0 as price.
                    if len(self.data[symbol]) > 0:
                        last_price = self.data[symbol][-1]
                        self.data[symbol].append(last_price)

    def Rebalance(self):
        # Performance / volatility data.
        performance_volatility = {}
        for symbol in self.symbols:
            if len(self.data[symbol]) == self.data[symbol].maxlen:
                back_adjusted_prices = [x for x in self.data[symbol]]
                performance = fk_tools.Return(back_adjusted_prices)
                
                back_adjusted_prices = back_adjusted_prices[-21:]
                volatility_1M = fk_tools.Volatility(back_adjusted_prices)
                performance_volatility[symbol] = [performance, volatility_1M]

        if len(performance_volatility) == 0: return
    
        # Performance sorting.
        long = [x for x in performance_volatility.items() if x[1][0] > 0]
        short = [x for x in performance_volatility.items() if x[1][0] < 0]
        
        # Volatility weighting.
        total_volatility_inversed = sum([(1 / x[1][1]) for x in long + short])
        if total_volatility_inversed == 0: return
        
        # count = len(long + short)
        count = len(long + short) * 2
        
        # Volatility targeting.
        portfolio_volatility = sum([((x[1][1]) / count) for x in long + short]) * sqrt(252)
        volatility_target_leverage = 2 * self.targeted_volatility / portfolio_volatility
 
        long_symbols = [x[0] for x in long]
        short_symbols = [x[0] for x in short]
        
        weight = {}
        for symbol_data in long + short:
            symbol = symbol_data[0]
            volatility = symbol_data[1][1]
            if volatility != 0:
                # 2x leverage - 100% long / 100% short.
                final_leverage = 2.0 * volatility_target_leverage
                # self.Log(f"Leverage: {final_leverage}")
                
                if symbol in long_symbols:
                    weight[symbol] = (final_leverage / volatility) / total_volatility_inversed
                else:
                    weight[symbol] = - (final_leverage / volatility) / total_volatility_inversed
            else: 
                weight[symbol] = 0

        # Trade execution.
        invested = [x.Key.Value for x in self.Portfolio if x.Value.Invested]
        for symbol in invested:
            if symbol not in long_symbols + short_symbols:
                self.Liquidate(symbol)
        
        for symbol, w in weight.items():
            self.SetHoldings(symbol, w)