加入arc和prr

api.py

@@ -59,8 +59,10 @@ class LoggerType(Enum):
 class SignalType(Enum):
     NORMAL = 1
 
+
 SignalType.NORMAL.p_type = PortfoliosType.NORMAL
 
+
 class DataSync(ABC):
     '''
     数据同步服务，需要同步数据的服务，可以实现该接口

asset_pool/asset_optimize.py

@@ -146,10 +146,14 @@ class FundDividendSortinoAssetOptimize(SortinoAssetOptimize):
     def get_groups(self):
         funds = pd.DataFrame(self.get_filtered_funds())
         result = []
+        if self.asset_include:
             include = list(self.asset_include.keys())[0]
             for key, fund_group in funds.groupby(by=include):
                 if key in self.asset_include[include]:
                     result.append(tuple(fund_group['id']))
+        else:
+            for (category, asset_type), fund_group in funds.groupby(by=['category', 'assetType']):
+                result.append(tuple(fund_group['id']))
         return result
 
     def get_pct_change(self, fund_ids, day):

config-svrobo6.yml

@@ -40,7 +40,7 @@ py-jftech:
       backtest: robo_executor.BacktestExecutor
       datum: basic.datum.DefaultDatum
       hold-report: portfolios.holder.DivHoldReportor
-      mpt: portfolios.builder.PoemPortfoliosBuilder
+      mpt: portfolios.builder.PoemARCPortfoliosBuilder
       dividend-holder: portfolios.holder.InvTrustPortfoliosHolder
   email:
     server: smtphz.qiye.163.com
@@ -50,14 +50,14 @@ py-jftech:
     max-workers: ${MAX_PROCESS:4}
 basic: # 基础信息模块
   sync:
-    start-date: 2007-01-01 # 同步数据开始日期
+    start-date: 2020-01-01 # 同步数据开始日期
   datum: # 资料模块
     change:
       date: ${DATUM_CHANGE_DATE}
       file: ${DATUM_CHANGE_FILE}
     excludes: # 排除的资料彭博ticker
-      backtest:
-        - 'TEMUSGI LX Equity'
+#      backtest:
+#        - 'TEMUSGI LX Equity'
       real:
         - 'FGFSACU LX Equity'
         - 'TEMUSGI LX Equity'
@@ -74,7 +74,7 @@ asset-pool: # 资产池模块
         weight: 0.3
       - years: 1
         weight: 0.2
-    asset-include: {'category':['US_STOCK','US_IG_BOND','US_HY_BOND']}
+    asset-include: {'customType':[1,2,3,4]}
     optimize-count: 3 #基金优选个数
 portfolios: # 投组模块
   holder: # 持仓投组相关
@@ -86,6 +86,9 @@ portfolios: # 投组模块
     warehouse-frequency: 1  #每隔1个月调一次仓
     redeem-list: [ 'TEUSAAU LX Equity', 'LIGTRAA ID Equity', 'TEMFHAC LX Equity', 'LUSHUAA ID Equity' ] #从持仓中的低风险资产“直接”按序赎回
   solver: # 解算器相关
+    model: prr # 结算模型 ARC ,PRR,  ~ 标准解算器
+    arc: on  #是否开启ARC
+    bRR: 0.01
     tol: 1E-10 # 误差满足条件
     navs: # 净值要求
       range: # 需要净值数据的区间, days: 90 表示90自然日，months: 3 表示3个自然月
@@ -93,21 +96,16 @@ portfolios: # 投组模块
       max-nan: # 最大缺失净值条件
         asset: 8 # 单一资产最多缺少多少交易日数据，则踢出资产池
         day: 0.5 # 单一交易日最多缺少百分之多少净值，则删除该交易日
-    normal-ratio: #US_STOCK：US_HY_BOND：US_IG_BOND三者分别对应低中高风险所占比率
-      US_STOCK: [ 0.5, 0.5, 0.7 ]
-      US_HY_BOND: [ 0.4, 0.4, 0.2 ]
-      US_IG_BOND: [ 0.1, 0.1, 0.1 ]
-    riskctl-ratio:
-      US_STOCK: [ 0.2, 0.4, 0.6 ]
-      US_HY_BOND: [ 0.5, 0.3, 0.1 ]
-      US_IG_BOND: [ 0.3, 0.3, 0.3 ]
+      risk: []  # 资产风险等级要求，可分开写也可以合并写，e.g. risk：[ 2, 3 ] 则表示 所有投组资产风险等级都是 2 或 3
+    LARC: [0, 0, 0, 0]  #低阈值
+    UARC: [1, 1, 1, 1] #高阈值
     matrix-rtn-days: 20 # 计算回报率矩阵时，回报率滚动天数
-    asset-count: [3,3] # 投组资产个数。e.g. count 或 [min, max] 分别表示 最大最小都为count 或 最小为min 最大为max，另外这里也可以类似上面给不同风险等级分别配置
+    asset-count: [5,5] # 投组资产个数。e.g. count 或 [min, max] 分别表示 最大最小都为count 或 最小为min 最大为max，另外这里也可以类似上面给不同风险等级分别配置
     mpt: # mpt计算相关
       cvar-beta: 0.2 # 计算Kbeta 需要用到
       quantile: 0.9 # 分位点，也可以给不同风险等级分别配置
       low-weight: 0.05 # 最低权重
-#      high-weight: [ 1 ] # 最高权重比例，可给一个值，也可以给多个值，当多个值时，第一个表示只有一个资产时权重，第二个表示只有两个资产时权重，以此类推，最后一个表示其他资产个数时的权重
+      high-weight: [ 0.35 ] # 最高权重比例，可给一个值，也可以给多个值，当多个值时，第一个表示只有一个资产时权重，第二个表示只有两个资产时权重，以此类推，最后一个表示其他资产个数时的权重
     poem: # poem相关
       cvar-scale-factor: 0.1 # 计算时用到的系数
 reports: # 报告模块相关
@@ -234,12 +232,12 @@ robo-executor: # 执行器相关
   use: ${ROBO_EXECUTOR:backtest} # 执行哪个执行器，优先取系统环境变量ROBO_EXECUTOR的值，默认backtest
   sync-data: ${SYNC_DATA:off} # 是否开启同步资料数据
   backtest: # 回测执行器相关
-    start-date: 2022-09-22 # 回测起始日期
-    end-date: 2023-07-03 # 回测截止日期
+    start-date: 2022-02-16 # 回测起始日期
+    end-date: 2023-01-03 # 回测截止日期
     sealing-period: 10 #调仓封闭期
-    start-step: ${BACKTEST_START_STEP:3} # 回测从哪一步开始执行 1:计算资产池；2:计算最优投组：3:计算再平衡信号以及持仓投组
+    start-step: ${BACKTEST_START_STEP:2} # 回测从哪一步开始执行 1:计算资产池；2:计算最优投组：3:计算再平衡信号以及持仓投组
     end-step: ${BACKTEST_END_STEP:3} # 回测从哪一步执行完成后结束执行 1:计算资产池；2:计算最优投组：3:计算再平衡信号以及持仓投组
-    clean-up: off
+    clean-up: on
   real: # 实盘执行器
     export: ${EXPORT_ENABLE:off} # 是否开启报告
     start-date: 2023-05-08 # 实盘开始时间

portfolios/builder.py

@@ -83,7 +83,7 @@ class MptPortfoliosBuilder(PortfoliosBuilder):
         rmp.delete(min_date=day, risk=risk)
 
 
-@component(bean_name='poem')
+@component(bean_name='mpt')
 class PoemPortfoliosBuilder(MptPortfoliosBuilder):
 
     def build_portfolio(self, day, type: PortfoliosType):
@@ -110,3 +110,88 @@ class PoemPortfoliosBuilder(MptPortfoliosBuilder):
         return result
 
 
+@component(bean_name='mpt')
+class MptARCPortfoliosBuilder(MptPortfoliosBuilder):
+
+    def get_portfolios(self, day, risk: PortfoliosRisk, type: PortfoliosType = PortfoliosType.NORMAL):
+        try:
+            portfolio = rmp.get_one(day, type, risk)
+            if not portfolio:
+                result, detail = self.build_portfolio(day, type)
+                for build_risk, datas in result.items():
+                    try:
+                        rmp.insert({
+                            **datas,
+                            'risk': build_risk,
+                            'type': type,
+                            'date': day
+                        })
+                    except IntegrityError as e:
+                        code, msg = e.args
+                        if code != constants.ER.DUP_ENTRY:
+                            raise e
+                portfolio = rmp.get_one(day, type, risk)
+            if SolveType(portfolio['solve']) is not SolveType.INFEASIBLE:
+                result = json.loads(portfolio['portfolio'])
+                return {int(x[0]): x[1] for x in result.items()}
+            return None
+        except Exception as e:
+            logger.exception(f"build protfolio of type[{type.name}] and risk[{risk.name}] with date[{format_date(day)}] failure.", exc_info=e)
+            raise e
+
+    def build_portfolio(self, day, type: PortfoliosType):
+        result = {}
+        detail = {}
+        risk = PortfoliosRisk.FT3
+        logger.info(
+            f"start to build protfolio of type[{type.name}] and risk[{risk.name}] with date[{format_date(day)}]")
+        solver = self._factory.create_solver(risk, type)
+        solver.reset_navs(day)
+        logger.debug({
+            'Khist': len(solver.rtn_history),
+            'beta': solver.get_config('mpt.cvar-beta'),
+            'Kbeta': solver.k_beta,
+        })
+        max_rtn, max_var, minCVaR_whenMaxR = solver.solve_max_rtn()
+        min_rtn, min_var, maxCVaR_whenMinV = solver.solve_min_rtn()
+        portfolio, cvar = solver.solve_mpt(min_rtn, max_rtn)
+        result[risk] = {
+            'solve': SolveType.MPT,
+            'portfolio': json.dumps(portfolio),
+            'cvar': cvar
+        } if portfolio else {
+            'solve': SolveType.INFEASIBLE
+        }
+        detail[risk] = {
+            'max_rtn': max_rtn,
+            'max_var': max_var,
+            'minCVaR_whenMaxR': minCVaR_whenMaxR,
+            'min_rtn': min_rtn,
+            'min_var': min_var,
+            'maxCVaR_whenMinV': maxCVaR_whenMinV,
+        }
+        return result, detail
+
+
+@component(bean_name='mpt')
+class PoemARCPortfoliosBuilder(MptARCPortfoliosBuilder):
+    def build_portfolio(self, day, type: PortfoliosType):
+        result, detail = super(PoemARCPortfoliosBuilder, self).build_portfolio(day, type)
+        risk = PortfoliosRisk.FT3
+        # if result[risk]['solve'] is SolveType.INFEASIBLE:
+        #     continue
+        solver = self._factory.create_solver(risk, type)
+        solver.reset_navs(day)
+        min_rtn = detail[risk]['min_rtn']
+        max_rtn = detail[risk]['max_rtn']
+        mpt_cvar = result[risk]['cvar']
+        maxCVaR_whenMinV = detail[risk]['maxCVaR_whenMinV']
+        portfolio, cvar = solver.solve_poem(min_rtn, max_rtn, mpt_cvar, maxCVaR_whenMinV)
+        if portfolio:
+            result[risk] = {
+                'solve': SolveType.POEM,
+                'portfolio': json.dumps(portfolio),
+                'cvar': cvar
+            }
+            detail[risk]['mpt_cvar'] = mpt_cvar
+        return result, detail

portfolios/holder.py

@@ -317,7 +317,7 @@ class InvTrustPortfoliosHolder(DividendPortfoliosHolder):
         day_div = sum(
             map(lambda k: share[k] * fund_dividend[k], filter(lambda k: k in fund_dividend, share.keys())))
         fund_dividend = last_nav['fund_div'] + day_div
-        # 基金净值+基金配息+产品配息
+        # todo 基金净值+基金配息+产品配息
         asset_nav = fund_av + fund_dividend + portfolio_div
         nav = last_nav['nav'] * asset_nav / last_nav['asset_nav']
         div_forecast = last_nav['div_forecast']

portfolios/solver.py

+import math
 import os
 import sys
 from logging import DEBUG, getLogger
 
+import numpy as np
 import pandas as pd
 from dateutil.relativedelta import relativedelta
 from numpy import NAN
-from py_jftech import component, autowired, get_config
+from py_jftech import component, autowired, get_config, filter_weekend
 from pyomo.environ import *
 
 from api import SolverFactory as Factory, PortfoliosRisk, PortfoliosType, AssetPool, Navs, Solver, Datum, DatumType
@@ -27,14 +29,26 @@ def create_solver():
 @component
 class DefaultFactory(Factory):
 
-    def create_solver(self, risk: PortfoliosRisk, type: PortfoliosType = PortfoliosType.NORMAL) -> Solver:
-        return DefaultSolver(risk, type)
+    def __init__(self):
+        self._config = get_config(__name__)
+
+    @property
+    def solver_model(self):
+        return self._config['model'].upper() if 'model' in self._config and self._config['model'] is not None else None
+
+    def create_solver(self, risk: PortfoliosRisk = None, type: PortfoliosType = PortfoliosType.NORMAL) -> Solver:
+        if self.solver_model == 'ARC':
+            return ARCSolver(type=type, risk=risk)
+        if self.solver_model == 'PRR':
+            if risk == PortfoliosRisk.FT3:
+                return PRRSolver(type=type, risk=risk)
+        return DefaultSolver(type=type, risk=risk)
 
 
 class DefaultSolver(Solver):
 
     @autowired
-    def __init__(self, risk: PortfoliosRisk, type: PortfoliosType, assets: AssetPool = None, navs: Navs = None,
+    def __init__(self, type: PortfoliosType, risk: PortfoliosRisk, assets: AssetPool = None, navs: Navs = None,
                  datum: Datum = None):
         self._category = None
         self._transfer_type = None
@@ -284,3 +298,159 @@ class DefaultSolver(Solver):
                 'port_CVaR': self.calc_port_cvar(model)
             })
             logger.debug('-------------------------------')
+
+
+class ARCSolver(DefaultSolver):
+
+    def __init__(self, type: PortfoliosType, risk: PortfoliosRisk, assets: AssetPool = None, navs: Navs = None,
+                 datum: Datum = None):
+        super().__init__(type, risk)
+        self.__date = None
+
+    @property
+    def date(self):
+        return self.__date
+
+    def calc_port_weight(self, model):
+        id_list = self.navs.columns
+        weight_list = [model.w[i]._value * model.z[i]._value for i in model.indices]
+        df_w = pd.DataFrame(data=weight_list, index=id_list, columns=['weight'])
+        df_w.replace(0, math.nan, inplace=True)
+        df_w.dropna(axis=0, inplace=True)
+        df_w['weight'] = pd.Series(format_weight(dict(df_w['weight'])))
+        dict_w = df_w.to_dict()['weight']
+        return dict_w
+
+    @property
+    def max_count(self):
+        count = self.get_config('asset-count')
+        return count[1] if isinstance(count, list) else count
+
+    @property
+    def min_count(self):
+        count = self.get_config('asset-count')
+        return min(count[0] if isinstance(count, list) else count, len(self.rtn_annualized))
+
+    def create_model(self):
+        low_weight = self.get_config('mpt.low-weight')
+        high_weight = self.get_config('mpt.high-weight')
+        if isinstance(high_weight, list):
+            high_weight = high_weight[min(len(self.navs.columns), self.min_count, len(high_weight)) - 1]
+
+        model = ConcreteModel()
+
+        model.indices = range(0, len(self.navs.columns))
+        model.w = Var(model.indices, domain=NonNegativeReals)
+        model.z = Var(model.indices, domain=Binary)
+        model.cons_sum_weight = Constraint(expr=sum([model.w[i] for i in model.indices]) == 1)
+        model.cons_num_asset = Constraint(
+            expr=inequality(self.min_count, sum([model.z[i] for i in model.indices]), self.max_count, strict=False))
+        model.cons_bounds_low = Constraint(model.indices, rule=lambda m, i: m.z[i] * low_weight <= m.w[i])
+        model.cons_bounds_up = Constraint(model.indices, rule=lambda m, i: m.z[i] * high_weight >= m.w[i])
+        if self._config['arc']:
+            LARC = self._config['LARC']
+            UARC = self._config['UARC']
+            numARC = len(LARC)  # this is the M in the doc
+            numAsset = len(self.navs.columns)
+
+            # This should from DB. We just fake value here for developing the code
+            datums = self._datum.get_datums(type=DatumType.FUND, datum_ids=list(self.navs.columns))
+            AssetARC = np.array([x['customType'] for x in datums], dtype=int)
+            # the above are input data from either config file or DB
+            # the following are POEM / MPT code
+
+            A = np.zeros((numARC, numAsset), dtype=int)
+            for i in range(numAsset):
+                A[AssetARC[i] - 1, i] = 1
+            model.cons_arc_low = Constraint(range(numARC),
+                                            rule=lambda m, i: LARC[i] <= sum([A[i, j] * m.w[j] for j in m.indices]))
+            model.cons_arc_up = Constraint(range(numARC),
+                                           rule=lambda m, i: UARC[i] >= sum([A[i, j] * m.w[j] for j in m.indices]))
+        return model
+
+    def reset_navs(self, day):
+        self.__date = filter_weekend(day)
+        asset_ids = self._assets.get_pool(self.date)
+        asset_risk = self.get_config('navs.risk')
+        datum = self._datum.get_datums(type=DatumType.FUND, datum_ids=asset_ids, risk=asset_risk)
+        exclude = self.get_config('navs.exclude-asset-type') or []
+        asset_ids = list(set(asset_ids) & set([x['id'] for x in datum if x['assetType'] not in exclude]))
+
+        min_date = self.date - relativedelta(**self.get_config('navs.range'))
+        navs = pd.DataFrame(self._navs.get_fund_navs(fund_ids=asset_ids, max_date=self.date, min_date=min_date))
+        navs = navs[navs['nav_date'].dt.day_of_week < 5]
+        navs['nav_date'] = pd.to_datetime(navs['nav_date'])
+        navs = navs.pivot_table(index='nav_date', columns='fund_id', values='nav_cal')
+        navs = navs.sort_index()
+
+        navs_nan = navs.isna().sum()
+        navs.drop(columns=[x for x in navs_nan.index if navs_nan.loc[x] >= self.get_config('navs.max-nan.asset')],
+                  inplace=True)
+        navs_nan = navs.apply(lambda r: r.isna().sum() / len(r), axis=1)
+        navs.drop(index=[x for x in navs_nan.index if navs_nan.loc[x] >= self.get_config('navs.max-nan.day')],
+                  inplace=True)
+        navs.fillna(method='ffill', inplace=True)
+        if navs.iloc[0].isna().sum() > 0:
+            navs.fillna(method='bfill', inplace=True)
+        self.set_navs(navs)
+
+
+class PRRSolver(ARCSolver):
+
+    def __init__(self, type: PortfoliosType, risk: PortfoliosRisk, assets: AssetPool = None, navs: Navs = None,
+                 datum: Datum = None):
+        super().__init__(type, risk)
+        self.__risk = None
+
+    def create_model(self):
+        model = super(PRRSolver, self).create_model()
+        # print(self.risks)
+        # 创建一个空列表来存储第二列的值
+        RR = []
+
+        # 遍历字典的键值对
+        for key, value in self.risks.items():
+            # 将值添加到列表中
+            RR.append(value)
+
+        # 打印第二列的值
+        # print(RR)
+        minRRweightWithinTRR = 0.7 + self._config['bRR']
+        TRR = 3
+        # RR = np.zeros(len(self.navs.columns), dtype=int)
+        # # Please note, RR should come from DB with real values. Here, we just assign fake values for coding
+        # for i in range(len(self.navs.columns)):
+        #     RR[i] = math.ceil((i + 1) / len(self.navs.columns) * 5)
+
+        # the following code are real model code ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+        model.cons_TRR = Constraint(expr=sum([model.w[i] * RR[i] for i in model.indices]) <= TRR)
+
+        RR_LE_TRR = np.zeros(len(self.navs.columns), dtype=int)
+        RR_in_1_5 = np.zeros(len(self.navs.columns), dtype=int)
+        RR_EQ_5 = np.zeros(len(self.navs.columns), dtype=int)
+        for i in range(len(self.navs.columns)):
+            if RR[i] <= TRR:
+                RR_LE_TRR[i] = 1
+
+            if RR[i] > 1 and RR[i] < 5:
+                RR_in_1_5[i] = 1
+            elif RR[i] == 5:
+                RR_EQ_5[i] = 1
+
+        model.cons_RR_LE_TRR = Constraint(
+            expr=sum([model.w[i] * RR_LE_TRR[i] for i in model.indices]) >= minRRweightWithinTRR)
+
+        if TRR < 5:
+            model.cons_RR_in_1_5 = Constraint(
+                expr=sum([model.z[i] * (RR_in_1_5[i] * self.max_count - RR_EQ_5[i]) for i in model.indices]) >= 0)
+        return model
+
+    def reset_navs(self, day):
+        super(PRRSolver, self).reset_navs(day=day)
+        datums = self._datum.get_datums(type=DatumType.FUND, datum_ids=list(self.navs.columns))
+        self.__risk = {x['id']: x['risk'] for x in datums}
+        # self.__risk = {x['risk'] for x in datums}
+
+    @property
+    def risks(self):
+        return self.__risk