| ... | ... | @@ -26,16 +26,18 @@ update_data_json --> update_data_json_province: 按省份分目录 |
|
|
|
|
|
|
|
## 更新量少的表直接入库
|
|
|
|
|
|
|
|
部署地址：192.168.109.45
|
|
|
|
|
|
|
|
project-collie-app/app_lake/data_pump/sync_online/ic_spider_update_lake.yml
|
|
|
|
|
|
|
|
## 更新量大的表写kafka队列然后入库
|
|
|
|
|
|
|
|
写kafka队列:
|
|
|
|
|
|
|
|
部署地址：192.168.109.45
|
|
|
|
project-collie-app/app_lake/data_pump/sync_online/ic_spider_update_lake.yml
|
|
|
|
|
|
|
|
读kafka更新mysql并写redis:
|
|
|
|
|
|
|
|
部署地址：192.168.109.200
|
|
|
|
project-collie-app/app_lake/data_pump/ic_lake/collie_ic_spider_update_lake_kafka.yml
|
|
|
|
|
|
|
|
|
| ... | ... | @@ -56,8 +58,9 @@ project-collie-app/app_lake/data_pump/ic_lake/collie_ic_spider_update_lake_kafka |
|
|
|
|
|
|
|
## 增量数据根据update_time从mysql获取
|
|
|
|
|
|
|
|
部署地址：192.168.109.200
|
|
|
|
project-collie-app/app_lake/data_pump/lake/lake_ic.yml
|
|
|
|
|
|
|
|
## 增量数据读redis查mysql获取
|
|
|
|
|
|
|
|
部署地址：192168.109.200
|
|
|
|
project-collie-app/app_lake/data_pump/ic_lake/collie_ic_spider_update_lake_redis.yml |
|
|
\ No newline at end of file |
