意义 FOF模式 投顾robo模式

fund_av	基金原始净值	sum（个股原始净值*对应份额）
fund_div	持有基金配息	sum（个股每股配息*对应份额）
fund_nav	基金复权净值	暂时不做此操作	不配息则配股
cash	现金（产品的现金账户）	"1.主动预留现金，用来配息
2.持有基金配息"	无现金预留
real_av	产品净值	fund_av + cash	fund_av
port_div	产品配息金额（产品主动配息）	使用预留现金进行配息	赎回特定基金进行配息
acc_div	产品累计配息（给到客户配息）	acc（port_div）	acc（port_div + fund_div）
acc_av	产品累计净值	real_av + acc_div	real_av + acc_div
nav	产品复权净值（配息再投资）	一般复权方法：real_av、port_div	将配息率设置为0

basic/navs.py

@@ -17,19 +17,20 @@ class DefaultNavs(Navs):
         navs = rfn.get_navs(fund_id=fund_ids, min_date=min_date, max_date=max_date)
         if navs and 'exrate' in self._config:
             navs = pd.DataFrame(navs)
-            navs = navs.pivot_table(index='nav_date', columns='fund_id', values='nav_cal')
             for exrate_config in self._config['exrate']:
-                exrate = pd.DataFrame(re.get_exrates(ticker=exrate_config['ticker'], min_date=navs.index.min(),
-                                                     max_date=navs.index.max()))
-                exrate = exrate[['date', 'close']]
-                exrate.set_index('date', inplace=True)
-                for fund in self._datum.get_datums(type=DatumType.FUND, crncy=exrate_config['from']):
-                    if fund['id'] in navs.columns:
-                        navs[fund['id']] = round(navs[fund['id']] * exrate['close'], 4)
-            navs = navs.reset_index().melt(id_vars='nav_date', value_name='nav_cal')
-            navs.dropna(inplace=True)
-            navs = navs[['fund_id', 'nav_date', 'nav_cal']]
-            navs.sort_values(by=['fund_id', 'nav_date'], inplace=True)
+                exrate = pd.DataFrame(re.get_exrates(ticker=exrate_config['ticker'], min_date=min_date,
+                                                     max_date=max_date))
+                exrate.rename(columns={'date': 'nav_date'}, inplace=True)
+                exrate = exrate[['nav_date', 'close']]
+                fund_ids = [x['id'] for x in self._datum.get_datums(type=DatumType.FUND, crncy=exrate_config['from'])]
+                merged = pd.merge(navs, exrate, on='nav_date', how='left')
+                merged.loc[merged['fund_id'].isin(fund_ids), 'nav_cal'] *= merged['close']
+                merged.loc[merged['fund_id'].isin(fund_ids), 'av'] *= merged['close']
+                merged.loc[merged['fund_id'].isin(fund_ids), 'dividend'] *= merged['close']
+                merged['nav_cal'] = merged['nav_cal'].round(4)  # 四舍五入保留四位小数
+                merged['av'] = merged['av'].round(4)  # 四舍五入保留四位小数
+                merged['dividend'] = merged['dividend'].round(4)  # 四舍五入保留四位小数
+                navs = merged.drop('close', axis=1)
             navs = navs.to_dict('records')
         return navs
 
@@ -84,4 +85,3 @@ class DefaultNavs(Navs):
             return red.get_last_one(eco_id=datum_id, max_date=max_date, by_release_date=by_release_date)
         else:
             return red.get_last(eco_id=datum_id, max_date=max_date, count=count, by_release_date=by_release_date)
-

basic/sync.py

@@ -54,7 +54,7 @@ class JDCDataSync(DataSync, ABC):
             while True:
                 url = self.build_urls(datum=datum, page=page, start_date=start_date)
                 if url is None:
-                    break
+                    raise Exception(f'''request data {datum['id']} not exist!''')
                 response = requests.get(url).json()
                 if not response['success']:
                     raise Exception(f'''request indictor failed: {response['status']}''')

config-svrobo5.yml

@@ -235,8 +235,8 @@ robo-executor: # 执行器相关
   use: ${ROBO_EXECUTOR:backtest} # 执行哪个执行器，优先取系统环境变量ROBO_EXECUTOR的值，默认backtest
   sync-data: ${SYNC_DATA:off} # 是否开启同步资料数据
   backtest: # 回测执行器相关
-    start-date: 2012-10-16 # 回测起始日期
-    end-date: 2023-03-01 # 回测截止日期
+    start-date: 2022-10-25 # 回测起始日期
+    end-date: 2023-06-01 # 回测截止日期
     sealing-period: 10 #调仓封闭期
     start-step: ${BACKTEST_START_STEP:1} # 回测从哪一步开始执行 1:计算资产池；2:计算最优投组：3:计算再平衡信号以及持仓投组
     end-step: ${BACKTEST_END_STEP:3} # 回测从哪一步执行完成后结束执行 1:计算资产池；2:计算最优投组：3:计算再平衡信号以及持仓投组

config-svrobo6.yml

@@ -51,7 +51,7 @@ py-jftech:
     max-workers: ${MAX_PROCESS:4}
 basic: # 基础信息模块
   sync:
-    start-date: 2020-01-01 # 同步数据开始日期
+    start-date: 1990-01-01 # 同步数据开始日期
   datum: # 资料模块
     change:
       date: ${DATUM_CHANGE_DATE}
@@ -62,10 +62,10 @@ basic: # 基础信息模块
       real:
         - 'FGFSACU LX Equity'
         - 'TEMUSGI LX Equity'
-#  navs: # 净值模块
-#    exrate: # 汇率，如果不开启，整个这块注释掉
-#      - from: EUR # 需要转换的货币类型
-#        ticker: EURUSD BGN Curncy # 汇率值的彭博ticker
+  navs: # 净值模块
+    exrate: # 汇率，如果不开启，整个这块注释掉
+      - from: EUR # 需要转换的货币类型
+        ticker: EURUSD BGN Curncy # 汇率值的彭博ticker
 asset-pool: # 资产池模块
   asset-optimize: # 资产优选模块
     sortino-weight: # sortino计算需要的权重，下面每一条为一次计算，e.g. months: 3, weight: 0.5 表示 3个月数据使用权重0.5来计算分值
@@ -87,7 +87,7 @@ portfolios: # 投组模块
     warehouse-frequency: 1  #每隔1个月调一次仓
     redeem-list: [ 'TEUSAAU LX Equity', 'LIGTRAA ID Equity', 'TEMFHAC LX Equity', 'LUSHUAA ID Equity' ] #从持仓中的低风险资产“直接”按序赎回
   solver: # 解算器相关
-    model: prr # 结算模型 ARC ,PRR,  ~ 标准解算器
+    model: arc # 结算模型 ARC ,PRR,  ~ 标准解算器
     arc: on  #是否开启ARC
     brr: 0.01 #误差补偿值
     trr: 3
@@ -232,12 +232,12 @@ reports: # 报告模块相关
         content: "Dear All: 附件是今天生成的监测数据，請驗收，謝謝！ 注>：該郵件為自動發送，如有問題請聯繫矽谷團隊 telan_qian@chifufund.com"
 robo-executor: # 执行器相关
   use: ${ROBO_EXECUTOR:backtest} # 执行哪个执行器，优先取系统环境变量ROBO_EXECUTOR的值，默认backtest
-  sync-data: ${SYNC_DATA:on} # 是否开启同步资料数据
+  sync-data: ${SYNC_DATA:off} # 是否开启同步资料数据
   backtest: # 回测执行器相关
     start-date: 2022-02-16 # 回测起始日期
     end-date: 2023-01-03 # 回测截止日期
     sealing-period: 10 #调仓封闭期
-    start-step: ${BACKTEST_START_STEP:2} # 回测从哪一步开始执行 1:计算资产池；2:计算最优投组：3:计算再平衡信号以及持仓投组
+    start-step: ${BACKTEST_START_STEP:3} # 回测从哪一步开始执行 1:计算资产池；2:计算最优投组：3:计算再平衡信号以及持仓投组
     end-step: ${BACKTEST_END_STEP:3} # 回测从哪一步执行完成后结束执行 1:计算资产池；2:计算最优投组：3:计算再平衡信号以及持仓投组
     clean-up: on
   real: # 实盘执行器

portfolios/dao/robo_hold_portfolios.py

@@ -6,13 +6,14 @@ __COLUMNS__ = {
     'rhp_id': 'id',
     'rhp_date': 'date',
     'rhp_risk': 'risk',
-    'rhp_div': 'dividend',
     'rhp_div_acc': 'div_acc',
     'rhp_rrs_id': 'signal_id',
     'rhp_rebalance': 'rebalance',
     'rhp_portfolios': 'portfolios',
     'rhp_nav': 'nav',
+    'rhp_cash': 'cash',
     'rhp_fund_av': 'fund_av',
+    'rhp_fund_nav': 'fund_nav',
     'rhp_fund_div': 'fund_div',
     'rhp_div_forecast': 'div_forecast',
     'rhp_asset_nav': 'asset_nav',

portfolios/holder.py

@@ -79,17 +79,23 @@ class DividendPortfoliosHolder(PortfoliosHolder):
 
     def do_rebalance(self, day, risk: PortfoliosRisk, signal, last_nav):
         weight = {int(x[0]): x[1] for x in json.loads(signal['portfolio']).items()}
-        dividend_acc = 0
-        fund_dividend = 0
         if last_nav:
             share = {int(x): y for x, y in json.loads(last_nav['portfolios'])['share'].items()}
+            share_nav = {int(x): y for x, y in json.loads(last_nav['portfolios'])['share'].items()}
             fund_div_tuple = self.get_navs_and_div(fund_ids=tuple(set(weight) | set(share)), day=day)
             navs = fund_div_tuple[0]
             fund_dividend = fund_div_tuple[1]
+            # 配息当天配股
+            for k in share_nav.keys():
+                if k in fund_dividend:
+                    share_nav[k] = (share_nav[k] * fund_dividend[k]) / (share_nav[k] * navs[k]) + share_nav[k]
             fund_dividend = sum(
                 map(lambda k: share[k] * fund_dividend[k], filter(lambda k: k in fund_dividend, share.keys())))
             dividend_acc = last_nav['div_acc']
-            fund_av = round(sum([navs[x] * y for x, y in share.items()]), 4) + last_nav['fund_div']
+            fund_av = round(sum([navs[x] * y for x, y in share.items()]), 4)
+            fund_nav = round(sum([navs[x] * y for x, y in share_nav.items()]), 4)
+            cash = last_nav['cash'] + fund_dividend
+            div_forecast = last_nav['div_forecast']
             #     每年的首个季度调整配息
             if day.month in self._config.get('dividend-adjust-day'):
                 asset_nav = last_nav['asset_nav']
@@ -100,84 +106,84 @@ class DividendPortfoliosHolder(PortfoliosHolder):
                         (self._config['dividend-rate'] - div_rate) / self._config['dividend-rate']) > \
                         self._config['dividend-drift-rate']:
                     # 以本月前一天的单位净值进行配息计算
-                    dividend = last_nav['asset_nav'] * self.month_dividend
-                else:
-                    dividend = last_nav['div_forecast']
-                fund_av = fund_av - dividend
-                dividend_acc = dividend + dividend_acc
-                div_forecast = dividend
-            else:
-                # 如果有未配息，则不再配息
-                if last_nav['dividend'] > 0:
-                    dividend = last_nav['dividend']
-                else:
-                    dividend = last_nav['div_forecast']
-                    fund_av = fund_av - last_nav['div_forecast']
-                    dividend_acc = last_nav['div_forecast'] + dividend_acc
-                div_forecast = dividend
-            asset_nav = fund_av + fund_dividend + dividend
+                    div_forecast = last_nav['asset_nav'] * self.month_dividend
+            asset_nav = fund_av + cash
             nav = last_nav['nav'] * asset_nav / last_nav['asset_nav']
             share = {x: fund_av * w / navs[x] for x, w in weight.items()}
+            share_nav = {x: fund_nav * w / navs[x] for x, w in weight.items()}
         else:
             fund_av = self.init_nav
             fund_div_tuple = self.get_navs_and_div(fund_ids=tuple(weight), day=day)
             navs = fund_div_tuple[0]
-            dividend = fund_av * self.month_dividend
-            div_forecast = dividend
-            fund_av = fund_av - dividend
-            dividend_acc = dividend + dividend_acc
+            fund_dividend = 0
+            cash = 0
+            div_forecast = fund_av * self.month_dividend
+            dividend_acc = 0
             nav = self.init_nav
-            asset_nav = fund_av + fund_dividend + dividend
+            asset_nav = fund_av + cash
             funds = self._datum.get_datums(type=DatumType.FUND)
             funds_subscription_rate = {fund['id']: fund.get('subscriptionRate', 0) for fund in funds}
             share = {x: (1 - funds_subscription_rate[x]) * (fund_av * w) / navs[x] for x, w in weight.items()}
+            share_nav = share
             #  初始买入扣手续费
             fee = sum(funds_subscription_rate[x] * (fund_av * w) for x, w in weight.items())
             fund_av = fund_av - fee
+            fund_nav = fund_av
         rhp.insert({
             'date': day,
             'risk': risk,
             'signal_id': signal['id'],
-            'dividend': dividend,
             'div_forecast': div_forecast if div_forecast else last_nav['div_forecast'] if last_nav else None,
             'fund_div': fund_dividend,
             'div_acc': dividend_acc,
             'rebalance': True,
             'portfolios': {
                 'weight': weight,
+                'weight_nav': weight,
                 'share': share,
+                'share_nav': share_nav,
             },
             'fund_av': fund_av,
+            'fund_nav': fund_nav,
             'nav': nav,
             'port_div': 0,
+            'cash': cash,
             'asset_nav': asset_nav,
         })
 
     def no_rebalance(self, day, risk: PortfoliosRisk, last_nav):
         share = {int(x): y for x, y in json.loads(last_nav['portfolios'])['share'].items()}
+        share_nav = {int(x): y for x, y in json.loads(last_nav['portfolios'])['share'].items()}
         fund_div_tuple = self.get_navs_and_div(fund_ids=tuple(share), day=day)
         navs = fund_div_tuple[0]
         fund_dividend = fund_div_tuple[1]
+        # 配息当天配股
+        for k in share_nav.keys():
+            if k in fund_dividend:
+                share_nav[k] = (share_nav[k] * fund_dividend[k]) / (share_nav[k] * navs[k]) + share_nav[k]
         fund_av = round(sum([navs[x] * y for x, y in share.items()]), 4)
+        fund_nav = round(sum([navs[x] * y for x, y in share_nav.items()]), 4)
         weight = {x: round(y * navs[x] / fund_av, 2) for x, y in share.items()}
+        weight_nav = {x: round(y * navs[x] / fund_av, 2) for x, y in share_nav.items()}
         weight = format_weight(weight)
-        dividend = last_nav['dividend']
+        weight_nav = format_weight(weight_nav)
         port_div = 0
-        fund_dividend = last_nav['fund_div'] + sum(
+        fund_dividend = sum(
             map(lambda k: share[k] * fund_dividend[k], filter(lambda k: k in fund_dividend, share.keys())))
         dividend_acc = last_nav['div_acc']
+        cash = last_nav['cash'] + fund_dividend
         if self.is_dividend_date(day):
-            port_div = dividend
-            asset_nav = fund_av + fund_dividend
-            dividend = 0
+            port_div = last_nav['div_forecast']
+            cash += port_div
+            asset_nav = fund_av + cash
+            dividend_acc += port_div
             nav = last_nav['nav'] * (asset_nav + port_div) / last_nav['asset_nav']
         else:
-            asset_nav = fund_av + fund_dividend + dividend
+            asset_nav = fund_av + cash
             nav = last_nav['nav'] * asset_nav / last_nav['asset_nav']
         rhp.insert({
             'date': day,
             'risk': risk,
-            'dividend': dividend,
             'div_forecast': last_nav['div_forecast'],
             'fund_div': fund_dividend,
             'div_acc': dividend_acc,
@@ -185,10 +191,14 @@ class DividendPortfoliosHolder(PortfoliosHolder):
             'rebalance': False,
             'portfolios': {
                 'weight': weight,
+                'weight_nav': weight_nav,
                 'share': share,
+                'share_nav': share_nav
             },
             'fund_av': fund_av,
+            'fund_nav': fund_nav,
             'nav': nav,
+            'cash': cash,
             'port_div': port_div,
             'asset_nav': asset_nav,
         })
@@ -235,22 +245,27 @@ class InvTrustPortfoliosHolder(DividendPortfoliosHolder):
         if last_nav:
             # 若非首次配息
             share = {int(x): y for x, y in json.loads(last_nav['portfolios'])['share'].items()}
+            share_nav = {int(x): y for x, y in json.loads(last_nav['portfolios'])['share_nav'].items()}
             fund_div_tuple = self.get_navs_and_div(fund_ids=tuple(set(weight) | set(share)), day=day)
             navs = fund_div_tuple[0]
             fund_dividend = fund_div_tuple[1]
-            day_div = sum(
+            # 配息当天配股
+            for k in share_nav.keys():
+                if k in fund_dividend:
+                    share_nav[k] = (share_nav[k] * fund_dividend[k]) / (share_nav[k] * navs[k]) + share_nav[k]
+            fund_dividend = sum(
                 map(lambda k: share[k] * fund_dividend[k], filter(lambda k: k in fund_dividend, share.keys())))
-            fund_dividend = last_nav['fund_div'] + day_div
-            dividend_acc = last_nav['div_acc']
+            dividend_acc = last_nav['div_acc'] + fund_dividend
             fund_av = round(sum([navs[x] * y for x, y in share.items()]), 4)
-            asset_nav = fund_av + fund_dividend
-            nav = last_nav['nav'] * asset_nav / last_nav['asset_nav']
+            fund_nav = round(sum([navs[x] * y for x, y in share_nav.items()]), 4)
+            asset_nav = fund_av
             share = {x: fund_av * w / navs[x] for x, w in weight.items()}
+            #  若调仓当日，有基金产生配息
+            share_nav = {x: fund_nav * w / navs[x] for x, w in weight.items()}
             if self.is_first_workday(day):
                 div_forecast = asset_nav * self.month_dividend
         else:
             fund_av = self.init_nav
-            nav = self.init_nav
             asset_nav = self.init_nav
             fund_div_tuple = self.get_navs_and_div(fund_ids=tuple(weight), day=day)
             navs = fund_div_tuple[0]
@@ -259,24 +274,29 @@ class InvTrustPortfoliosHolder(DividendPortfoliosHolder):
             funds = self._datum.get_datums(type=DatumType.FUND)
             funds_subscription_rate = {fund['id']: fund.get('subscriptionRate', 0) for fund in funds}
             share = {x: (1 - funds_subscription_rate[x]) * (fund_av * w) / navs[x] for x, w in weight.items()}
+            share_nav = share
             #  初始买入扣手续费
             fee = sum(funds_subscription_rate[x] * (fund_av * w) for x, w in weight.items())
             fund_av = fund_av - fee
+            fund_nav = fund_av
+
         rhp.insert({
             'date': day,
             'risk': risk,
             'signal_id': signal['id'],
-            'dividend': 0,
             'fund_div': fund_dividend,
             'div_forecast': div_forecast if div_forecast else last_nav['div_forecast'] if last_nav else None,
             'div_acc': dividend_acc,
             'rebalance': True,
             'portfolios': {
                 'weight': weight,
+                'weight_nav': weight,
                 'share': share,
+                'share_nav': share_nav,
             },
             'fund_av': fund_av,
-            'nav': nav,
+            'fund_nav': fund_nav,
+            'nav': 0,
             'port_div': 0,
             'asset_nav': asset_nav,
         })
@@ -291,43 +311,37 @@ class InvTrustPortfoliosHolder(DividendPortfoliosHolder):
         port_div = 0
         dividend_acc = last_nav['div_acc']
         share = {int(x): y for x, y in json.loads(last_nav['portfolios'])['share'].items()}
+        share_nav = {int(x): y for x, y in json.loads(last_nav['portfolios'])['share_nav'].items()}
         fund_div_tuple = self.get_navs_and_div(fund_ids=tuple(share), day=day)
         navs = fund_div_tuple[0]
         fund_dividend = fund_div_tuple[1]
+        # 配息当天配股
+        for k in share_nav.keys():
+            if k in fund_dividend:
+                share_nav[k] = (share_nav[k] * fund_dividend[k]) / (share_nav[k] * navs[k]) + share_nav[k]
         # 配息日当天取得调仓日计算的应调仓金额,做实际份额赎回,这里的金额(即月初计算的赎回金额)用于转换成“赎回目标的份额”
         need_div = last_nav['div_forecast']
-        portfolio_div = 0
         if self.is_dividend_date(day) and need_div > 0:
             funds = self._datum.get_datums(type=DatumType.FUND, ticker=self._config['redeem-list'])
-            # 获取需要配息的金额
-            for fund in funds:
-                if fund['id'] in share.keys():
-                    #  按配息金额依次扣除对应基金份额
-                    if share[fund['id']] * navs[fund['id']] <= need_div:
-                        share[fund['id']] = 0
-                        need_div = need_div - share[fund['id']] * navs[fund['id']]
-                    else:
-                        share[fund['id']] = (share[fund['id']] * navs[fund['id']] - need_div) / navs[fund['id']]
-                        break
+            self.exec_redeem(funds, navs, need_div, share)
+            self.exec_redeem(funds, navs, need_div, share_nav)
             port_div = last_nav['div_forecast']
-            dividend_acc = dividend_acc + port_div
-            portfolio_div = port_div
         fund_av = round(sum([navs[x] * y for x, y in share.items()]), 4)
+        fund_nav = round(sum([navs[x] * y for x, y in share_nav.items()]), 4)
         weight = {x: round(y * navs[x] / fund_av, 2) for x, y in share.items()}
+        weight_nav = {x: round(y * navs[x] / fund_av, 2) for x, y in share_nav.items()}
         weight = format_weight(weight)
-        day_div = sum(
+        weight_nav = format_weight(weight_nav)
+        fund_dividend = sum(
             map(lambda k: share[k] * fund_dividend[k], filter(lambda k: k in fund_dividend, share.keys())))
-        fund_dividend = last_nav['fund_div'] + day_div
-        # todo 基金净值+基金配息+产品配息
-        asset_nav = fund_av + fund_dividend + portfolio_div
-        nav = last_nav['nav'] * asset_nav / last_nav['asset_nav']
+        dividend_acc = dividend_acc + port_div + fund_dividend
+        asset_nav = fund_av
         div_forecast = last_nav['div_forecast']
         if self.is_first_workday(day):
             div_forecast = asset_nav * self.month_dividend
         rhp.insert({
             'date': day,
             'risk': risk,
-            'dividend': 0,
             'fund_div': fund_dividend,
             'div_forecast': div_forecast,
             'div_acc': dividend_acc,
@@ -335,14 +349,29 @@ class InvTrustPortfoliosHolder(DividendPortfoliosHolder):
             'rebalance': False,
             'portfolios': {
                 'weight': weight,
+                'weight_nav': weight_nav,
                 'share': share,
+                'share_nav': share_nav
             },
             'fund_av': fund_av,
-            'nav': nav,
+            'fund_nav': fund_nav,
+            'nav': 0,
             'port_div': port_div,
             'asset_nav': asset_nav,
         })
 
+    def exec_redeem(self, funds, navs, need_div, share):
+        # 获取需要配息的金额
+        for fund in funds:
+            if fund['id'] in share.keys():
+                #  按配息金额依次扣除对应基金份额
+                if share[fund['id']] * navs[fund['id']] <= need_div:
+                    share[fund['id']] = 0
+                    need_div = need_div - share[fund['id']] * navs[fund['id']]
+                else:
+                    share[fund['id']] = (share[fund['id']] * navs[fund['id']] - need_div) / navs[fund['id']]
+                    break
+
 
 @component(bean_name='hold-report')
 class DivHoldReportor(RoboReportor):
@@ -355,7 +384,6 @@ class DivHoldReportor(RoboReportor):
         holds = pd.DataFrame(rhp.get_list(max_date=max_date, min_date=min_date))
         if not holds.empty:
             holds['signal_type'] = 'INIT'
-            holds['cash'] = holds['dividend'] + holds['fund_div']
             holds['real_av'] = holds['asset_nav']
             holds = holds[
                 ['date', 'signal_type', 'fund_av', 'fund_div', 'cash', 'real_av', 'port_div', 'acc_av', 'nav']]