from clr import AddReference
AddReference("System")
AddReference("QuantConnect.Algorithm")
AddReference("QuantConnect.Algorithm.Framework")
AddReference("QuantConnect.Common")
AddReference("QuantConnect.Indicators")

from System import *
from QuantConnect import *
from QuantConnect.Indicators import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework import *
from QuantConnect.Algorithm.Framework.Portfolio import *
from MeanCVaRPortfolioOptimizer import MeanCVaRPortfolioOptimizer
from datetime import timedelta
import numpy as np
import pandas as pd

### <summary>
### Provides an implementation of Mean-CVaR portfolio optimization adjusting modern portfolio theory's risk measure to coherent conditional value at risk (CVaR).
### The default model uses the MinimumCVaRPortfolioOptimizer that accepts a 63-row matrix of 1-day returns.
### </summary>
class MeanCVaROptimizationPortfolioConstructionModel(PortfolioConstructionModel):
    def __init__(self,
                 rebalance = Resolution.Daily,
                 portfolioBias = PortfolioBias.Long,
                 lookback = 1,
                 period = 63,
                 resolution = Resolution.Daily,
                 targetReturn = None,
                 optimizer = None,
                 alpha = 0.05,
                 riskAverse = 3.):
        """Initialize the model
        Args:
            rebalance: Rebalancing parameter. If it is a timedelta, date rules or Resolution, it will be converted into a function.
                              If None will be ignored.
                              The function returns the next expected rebalance time for a given algorithm UTC DateTime.
                              The function returns null if unknown, in which case the function will be called again in the
                              next loop. Returning current time will trigger rebalance.
            portfolioBias: Specifies the bias of the portfolio (Long only for convex CVaR optimization)
            lookback(int): Historical return lookback period
            period(int): The time interval of history price to calculate the weight
            resolution: The resolution of the history price
            optimizer(class): Method used to compute the portfolio weights
            alpha(float): Percentage CVaR using
            riskAverse: tradeoff mean CVaR weighing"""
        self.lookback = lookback
        self.period = period
        self.resolution = resolution
        self.portfolioBias = PortfolioBias.Long
        self.sign = lambda x: -1 if x < 0 else (1 if x > 0 else 0)

        lower = 0 
        upper = 1
        self.optimizer = MeanCVaRPortfolioOptimizer(lower, upper, targetReturn) if optimizer is None else optimizer
        self.alpha = alpha
        self.riskAverse = riskAverse

        self.symbolDataBySymbol = {}

        # If the argument is an instance of Resolution or Timedelta
        # Redefine rebalancingFunc
        rebalancingFunc = rebalance
        if isinstance(rebalance, int):
            rebalance = Extensions.ToTimeSpan(rebalance)
        if isinstance(rebalance, timedelta):
            rebalancingFunc = lambda dt: dt + rebalance
        if rebalancingFunc:
            self.SetRebalancingFunc(rebalancingFunc)

    def ShouldCreateTargetForInsight(self, insight):
        if len(PortfolioConstructionModel.FilterInvalidInsightMagnitude(self.Algorithm, [insight])) == 0:
            return False

        symbolData = self.symbolDataBySymbol.get(insight.Symbol)
        if insight.Magnitude is None:
            self.algorithm.SetRunTimeError(ArgumentNullException('MeanCVaROptimizationPortfolioConstructionModel does not accept \'None\' as Insight.Magnitude. Please checkout the selected Alpha Model specifications.'))
            return False
        if symbolData is None:                      # empty history will not be added so shall be dropped
            return False
        symbolData.Add(self.Algorithm.Time, insight.Magnitude)

        return True

    def DetermineTargetPercent(self, activeInsights):
        """
         Will determine the target percent for each insight
        Args:
        Returns:
        """
        targets = {}

        # If we have no insights just return an empty target list
        if len(activeInsights) == 0:
            return targets

        symbols = [insight.Symbol for insight in activeInsights]

        # Create a dictionary keyed by the symbols in the insights with an pandas.Series as value to create a data frame
        returns = { str(symbol) : data.Return for symbol, data in self.symbolDataBySymbol.items() if symbol in symbols }
        returns = pd.DataFrame(returns)
        
        # The portfolio optimizer finds the optional weights for the given data
        weights = self.optimizer.Optimize(returns, self.alpha, self.riskAverse)
        weights = pd.Series(weights, index = returns.columns)
        
        # Create portfolio targets from the specified insights
        for insight in activeInsights:
            weight = weights[str(insight.Symbol)]

            # don't trust the optimizer
            if self.portfolioBias != PortfolioBias.Long and self.sign(weight) != self.portfolioBias:
                weight = 0
            targets[insight] = weight

        return targets

    def OnSecuritiesChanged(self, algorithm, changes):
        '''Event fired each time the we add/remove securities from the data feed
        Args:
            algorithm: The algorithm instance that experienced the change in securities
            changes: The security additions and removals from the algorithm'''

        # clean up data for removed securities
        super().OnSecuritiesChanged(algorithm, changes)
        for removed in changes.RemovedSecurities:
            if removed.Symbol in self.symbolDataBySymbol:
                symbolData = self.symbolDataBySymbol.pop(removed.Symbol, None)
                symbolData.Reset()

        # initialize data for added securities
        symbols = [ x.Symbol for x in changes.AddedSecurities ]
        history = algorithm.History(symbols, self.lookback * self.period, self.resolution)
        if history.empty: return

        tickers = history.index.levels[0]
        for ticker in tickers:
            symbol = SymbolCache.GetSymbol(ticker)

            if symbol not in self.symbolDataBySymbol:
                symbolData = self.MeanCVaRSymbolData(symbol, self.lookback, self.period)
                symbolData.WarmUpIndicators(history.loc[ticker].iloc[:-1])                  # The last one will use present value which added later to avoid same index
                self.symbolDataBySymbol[symbol] = symbolData

    class MeanCVaRSymbolData:
        '''Contains data specific to a symbol required by this model'''
        def __init__(self, symbol, lookback, period):
            self.symbol = symbol
            self.roc = RateOfChange(f'{symbol}.ROC({lookback})', lookback)
            self.roc.Updated += self.OnRateOfChangeUpdated
            self.window = RollingWindow[IndicatorDataPoint](period)

        def Reset(self):
            self.roc.Updated -= self.OnRateOfChangeUpdated
            self.roc.Reset()
            self.window.Reset()

        def WarmUpIndicators(self, history):
            for tuple in history.itertuples():
                self.roc.Update(tuple.Index, tuple.close)

        def OnRateOfChangeUpdated(self, roc, value):
            if roc.IsReady:
                self.window.Add(value)

        def Add(self, time, value):
            item = IndicatorDataPoint(self.symbol, time, value)
            self.window.Add(item)

        @property
        def Return(self):
            return pd.Series(
                data = [(1 + float(x.Value))**252 - 1 for x in self.window],
                index = [x.EndTime for x in self.window])

        @property
        def IsReady(self):
            return self.window.IsReady

        def __str__(self, **kwargs):
            return '{}: {:.2%}'.format(self.roc.Name, (1 + self.window[0])**252 - 1)

import numpy as np
import pandas as pd
import cvxpy as cp

### <summary>
### Provides an implementation of a portfolio optimizer that calculate the optimal weights 
### with the weight range from 0 to 1 and minimize the portfolio conditional value at risk (CVaR)
### </summary>
class MeanCVaRPortfolioOptimizer:
    '''Provides an implementation of a portfolio optimizer that calculate the optimal weights 
    with the weight range from 0 to 1 and minimize the portfolio CVaR'''
    def __init__(self, 
                 minimum_weight = 0, 
                 maximum_weight = 1, 
                 target_return = None):
        '''Initialize the MinimumCVaRPortfolioOptimizer
        Args:
            minimum_weight(float): The lower bounds on portfolio weights, non-negative value
            maximum_weight(float): The upper bounds on portfolio weights
            target_return(float): The target portfolio return'''
        self.minimum_weight = max(0, minimum_weight)
        self.maximum_weight = maximum_weight
        self.target_return = target_return

    def Optimize(self, historicalReturns, alpha = 0.05, riskAverse = 3, expectedReturns = None):
        '''
        Perform portfolio optimization for a provided matrix of historical returns and an array of expected returns
        args:
            historicalReturns: Matrix of annualized historical returns where each column represents a security and each row returns for the given date/time (size: K x N).
            alpha: percentage CVaR using, default 5% CVaR
            riskAverse: risk aversion level
            expectedReturns: Array of double with the portfolio annualized expected returns (size: K x 1).
        Returns:
            Array of double with the portfolio weights (size: K x 1)
        '''
        if expectedReturns is None:
            expectedReturns = historicalReturns.mean(axis=0).values

        size = int(historicalReturns.shape[1])   # K x 1
        number = int(historicalReturns.shape[0])    # 1 x N
        x0 = np.array(size * [1. / size])
        
        # https://richtarik.org/docs/Kisiala_Dissertation.pdf, p.27, eq. 3.14
        t, z, weights = cp.Variable(), cp.Variable((number, 1), nonneg=True), cp.Variable((size, 1), nonneg=True)
        
        objective = cp.Maximize(expectedReturns @ weights - riskAverse * (t + cp.sum(z)/(number * (1 - alpha))))
        
        constrains = [z >= - historicalReturns.values @ weights - t] \
                        + self.get_budget_constraint(weights)\
                        + self.get_boundary_conditions(weights)
                        
        if self.target_return is not None:
            constrains = constrains + self.get_target_constraint(self, weights, expectedReturns)
                        
        problem = cp.Problem(objective, constrains)
        problem.solve(solver="SCS")
        
        weights = weights.value
        
        return weights.flatten() if weights is not None else x0
        
    def get_boundary_conditions(self, weights):
        '''Creates the boundary condition for the portfolio weights'''
        return [weights >= self.minimum_weight, weights <= self.maximum_weight]

    def get_budget_constraint(self, weights):
        '''Defines a budget constraint: the sum of the weights equals unity'''
        return [cp.sum(weights) == 1.]
        
    def get_target_constraint(self, weights, expectedReturns):
        '''Ensure that the portfolio return target a given return'''
        return [expectedReturns @ weights >= self.target_return]

from MeanCVaROptimizationPortfolioConstructionModel import MeanCVaROptimizationPortfolioConstructionModel

class CreativeGreenKoala(QCAlgorithm):

    def Initialize(self):
        self.SetStartDate(2021, 5, 1)
        self.SetCash(100000) 
        
        self.AddUniverse(self.MyCoarseFilterFunction)
        self.UniverseSettings.Resolution = Resolution.Daily
        self.SetSecurityInitializer(lambda x: x.SetDataNormalizationMode(DataNormalizationMode.Raw))
        
        self.SetPortfolioConstruction(MeanCVaROptimizationPortfolioConstructionModel())
        self.SetExecution(ImmediateExecutionModel())

    def OnData(self, data):
        insights = []
        
        df = self.History(self.symbols, 252, Resolution.Daily)
        if df.empty: return
        df = df.close.unstack("symbol").pct_change()[1:].dropna(axis=1)
        
        # required to include magnitude to use (expected return)
        [insights.append(Insight.Price(df.columns[n], Expiry.EndOfDay, InsightDirection.Up, df[df.columns[n]].mean(axis=0), None, None, None)) for n in range(len(df.columns))]
        self.EmitInsights(insights)
           
    def MyCoarseFilterFunction(self, coarse):
        sortedByDollarVolume = sorted(coarse, key=lambda x: x.DollarVolume, reverse=True)
        self.symbols = [ x.Symbol for x in sortedByDollarVolume 
                    if x.Price > 10 and x.DollarVolume > 10000000 ][:10]
        return self.symbols