mirror of
https://github.com/ProudMuBai/GoFilm.git
synced 2026-02-03 22:44:47 +08:00
failture record recover
This commit is contained in:
mubai
@@ -14,7 +14,7 @@
|
||||
"@videojs-player/vue": "^1.0.0",
|
||||
"axios": "^1.3.4",
|
||||
"element-plus": "^2.8.6",
|
||||
"video.js": "^8.0.4",
|
||||
"video.js": "^7.0.0",
|
||||
"vue": "^3.4.31"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
@@ -167,7 +167,7 @@ button:focus-visible {
|
||||
}
|
||||
}
|
||||
|
||||
.el-dialog {
|
||||
.el-dialog, .el-message-box {
|
||||
border-radius: 8px !important;
|
||||
background-image: linear-gradient(135deg, #81FFEF 10%, #F067B4 100%) !important;
|
||||
--el-text-color-primary: #5e1e99b8;
|
||||
@@ -188,6 +188,13 @@ button:focus-visible {
|
||||
border: none;
|
||||
}
|
||||
|
||||
/* messagebox样式 */
|
||||
.el-message-box {
|
||||
--el-color-primary: #9b49e7;
|
||||
--el-border-color: #f9f9f9;
|
||||
}
|
||||
|
||||
|
||||
/*分页插件颜色*/
|
||||
.el-popper {
|
||||
--el-color-primary: var(--paging-parmary-color);
|
||||
|
||||
@@ -1,45 +1,128 @@
|
||||
<template>
|
||||
<div class="container">
|
||||
|
||||
<div class="params_form">
|
||||
<el-form :model="data.params" class="cus_form">
|
||||
<el-form-item>
|
||||
<el-select v-model="data.params.originId" placeholder="采集来源">
|
||||
<el-option
|
||||
v-for="item in data.options.origin"
|
||||
:key="item.value"
|
||||
:label="item.name"
|
||||
:value="item.value"
|
||||
/>
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
<el-form-item v-if="false">
|
||||
<el-select v-model="data.params.collectType" placeholder="采集类型">
|
||||
<el-option
|
||||
v-for="item in data.options.collectType"
|
||||
:key="item.value"
|
||||
:label="item.name"
|
||||
:value="item.value"
|
||||
/>
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
<el-form-item>
|
||||
<el-select v-model="data.params.status" placeholder="记录状态">
|
||||
<el-option
|
||||
v-for="item in data.options.status"
|
||||
:key="item.value"
|
||||
:label="item.name"
|
||||
:value="item.value"
|
||||
/>
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
<el-form-item>
|
||||
<el-date-picker
|
||||
v-model="data.dateGroup"
|
||||
value-format="YYYY-MM-DD HH:mm:ss"
|
||||
type="datetimerange"
|
||||
start-placeholder="起始时间"
|
||||
end-placeholder="终止时间"
|
||||
/>
|
||||
</el-form-item>
|
||||
<el-form-item>
|
||||
<el-button type="primary" @click="filterRecord">查询</el-button>
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
</div>
|
||||
<div class="content">
|
||||
<el-table
|
||||
:data="data.records" style="width: 100%" border size="default"
|
||||
table-layout="auto" max-height="calc(68vh - 20px)"
|
||||
row-key="id" fit
|
||||
:row-class-name="'cus-tr'">
|
||||
<el-table-column type="index" align="left" min-width="35px" label="序列">
|
||||
:data="data.records"
|
||||
style="width: 100%"
|
||||
border
|
||||
size="default"
|
||||
table-layout="auto"
|
||||
max-height="calc(68vh - 20px)"
|
||||
row-key="id"
|
||||
fit
|
||||
:row-class-name="'cus-tr'"
|
||||
>
|
||||
<el-table-column
|
||||
type="index"
|
||||
align="left"
|
||||
min-width="35px"
|
||||
label="序列"
|
||||
>
|
||||
<template #default="scope">
|
||||
<span style="color: #8b40ff">{{ scope.row.ID }}</span>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="originId" align="center" label="采集站">
|
||||
<template #default="scope">
|
||||
<el-tag type="primary" disable-transitions>{{ scope.row.originName }}</el-tag>
|
||||
<el-tag type="primary" disable-transitions
|
||||
>{{ scope.row.originName }}
|
||||
</el-tag>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="originId" align="center" min-width="100px" label="采集源ID">
|
||||
<el-table-column
|
||||
prop="originId"
|
||||
align="center"
|
||||
min-width="100px"
|
||||
label="采集源ID"
|
||||
>
|
||||
<template #default="scope">
|
||||
<el-tag type="success" disable-transitions>{{ scope.row.originId }}</el-tag>
|
||||
<el-tag type="success" disable-transitions
|
||||
>{{ scope.row.originId }}
|
||||
</el-tag>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="collectType" align="center" label="采集类型" show-overflow-tooltip>
|
||||
<el-table-column
|
||||
prop="collectType"
|
||||
align="center"
|
||||
label="采集类型"
|
||||
show-overflow-tooltip
|
||||
>
|
||||
<template #default="scope">
|
||||
<el-tag type="success" disable-transitions>{{ scope.row.collectType == 0 ? '影片详情' : '未知' }}</el-tag>
|
||||
<el-tag type="success" disable-transitions
|
||||
>{{ scope.row.collectType == 0 ? "影片详情" : "未知" }}
|
||||
</el-tag>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="pageNumber" align="center" label="分页页码">
|
||||
<template #default="scope">
|
||||
<el-tag type="warning" disable-transitions>{{ scope.row.pageNumber }}</el-tag>
|
||||
<el-tag type="warning" disable-transitions
|
||||
>{{ scope.row.pageNumber }}
|
||||
</el-tag>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="hour" align="center" label="采集时长">
|
||||
<template #default="scope">
|
||||
<el-tag type="warning" disable-transitions>{{ scope.row.hour }}</el-tag>
|
||||
<el-tag type="warning" disable-transitions
|
||||
>{{ scope.row.hour }}
|
||||
</el-tag>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="cause" align="center" label="失败原因" min-width="150px" >
|
||||
<el-table-column
|
||||
prop="cause"
|
||||
align="center"
|
||||
label="失败原因"
|
||||
min-width="150px"
|
||||
>
|
||||
<template #default="scope">
|
||||
<el-tag type="danger" disable-transitions>{{ scope.row.cause }}</el-tag>
|
||||
<el-tag type="danger" disable-transitions
|
||||
>{{ scope.row.cause }}
|
||||
</el-tag>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="status" align="center" label="状态">
|
||||
@@ -48,105 +131,249 @@
|
||||
<el-tag v-else type="success">已处理</el-tag>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="UpdatedAt" align="center" label="执行时间" min-width="100px" >
|
||||
<el-table-column
|
||||
prop="UpdatedAt"
|
||||
align="center"
|
||||
label="执行时间"
|
||||
min-width="100px"
|
||||
>
|
||||
<template #default="scope">
|
||||
<el-tag :type="`${scope.row.status == 1 ? 'warning':'success' }`" disable-transitions>{{ scope.row.timeFormat }}</el-tag>
|
||||
<el-tag
|
||||
:type="`${scope.row.status == 1 ? 'warning' : 'success'}`"
|
||||
disable-transitions
|
||||
>{{ scope.row.timeFormat }}
|
||||
</el-tag>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column label="操作" align="center" min-width="100px">
|
||||
<template #default="scope">
|
||||
<el-tooltip content="采集重试" placement="top"><el-button type="success" :icon="RefreshRight" @click="collectRecover(scope.row.ID)" plain circle/></el-tooltip>
|
||||
<el-tooltip content="删除记录" placement="top"><el-button type="danger" :icon="Delete" @click="delRecord(scope.row.ID)" plain circle/></el-tooltip>
|
||||
<el-tooltip content="采集重试" placement="top">
|
||||
<el-button
|
||||
type="success"
|
||||
:icon="RefreshRight"
|
||||
@click="collectRecover(scope.row.ID)"
|
||||
plain
|
||||
circle
|
||||
/>
|
||||
</el-tooltip>
|
||||
<el-tooltip v-if="false" content="删除记录" placement="top">
|
||||
<el-button
|
||||
type="danger"
|
||||
:icon="Delete"
|
||||
@click="delRecord(scope.row.ID)"
|
||||
plain
|
||||
circle
|
||||
/>
|
||||
</el-tooltip>
|
||||
</template>
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
<div class="pagination">
|
||||
<el-pagination :page-sizes="[10, 20, 50, 100, 500]" background layout="prev, pager, next, sizes, total, jumper"
|
||||
:total="data.page.total" v-model:page-size="data.page.pageSize"
|
||||
<div class="cus_util">
|
||||
<el-tooltip content="重试采集所有失败记录" placement="top">
|
||||
<el-button
|
||||
color="#d942bf"
|
||||
:icon="HelpFilled"
|
||||
@click="collectRecoverAll"
|
||||
>RetryAll</el-button
|
||||
>
|
||||
</el-tooltip>
|
||||
<el-tooltip content="清除已处理记录,保留未处理记录" placement="top">
|
||||
<el-button
|
||||
type="warning"
|
||||
:icon="WarningFilled"
|
||||
@click="cleanDoneRecord"
|
||||
>CleanDone</el-button
|
||||
>
|
||||
</el-tooltip>
|
||||
<el-tooltip content="清除所有记录" placement="top">
|
||||
<el-button type="danger" :icon="BellFilled" @click="cleanAllRecord"
|
||||
>CleanAll</el-button
|
||||
>
|
||||
</el-tooltip>
|
||||
</div>
|
||||
<el-pagination
|
||||
:page-sizes="[10, 20, 50, 100, 500]"
|
||||
background
|
||||
layout="prev, pager, next, sizes, total, jumper"
|
||||
:total="data.page.total"
|
||||
v-model:page-size="data.page.pageSize"
|
||||
v-model:current-page="data.page.current"
|
||||
@change="getRecords" hide-on-single-page/>
|
||||
@change="getRecords"
|
||||
hide-on-single-page
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
|
||||
import {Aim, Delete, Edit, RefreshRight} from "@element-plus/icons-vue";
|
||||
import {fmt} from "../../../utils/format";
|
||||
import {onMounted, reactive} from "vue";
|
||||
import {ApiGet} from "../../../utils/request";
|
||||
import {ElMessage} from "element-plus";
|
||||
|
||||
import {
|
||||
Delete,
|
||||
RefreshRight,
|
||||
BellFilled,
|
||||
WarningFilled,
|
||||
HelpFilled,
|
||||
} from "@element-plus/icons-vue";
|
||||
import { fmt } from "../../../utils/format";
|
||||
import { onMounted, reactive } from "vue";
|
||||
import { ApiGet } from "../../../utils/request";
|
||||
import { ElMessage, ElMessageBox } from "element-plus";
|
||||
|
||||
const data = reactive({
|
||||
records: [],
|
||||
page: {current: 1, pageCount: 0, pageSize: 10, total: 0},
|
||||
params: {},
|
||||
})
|
||||
|
||||
page: { current: 1, pageCount: 0, pageSize: 10, total: 0 },
|
||||
params: {
|
||||
originId: "",
|
||||
collectType: -1,
|
||||
status: -1,
|
||||
betweenTime: "",
|
||||
endTime: "",
|
||||
},
|
||||
dateGroup: [],
|
||||
options: { origin: [], collectType: [], status: [] },
|
||||
});
|
||||
|
||||
// 获取影片分页信息
|
||||
const getRecords = () => {
|
||||
let {current, pageSize} = data.page
|
||||
let params = data.params
|
||||
ApiGet(`/manage/collect/record/list`, {...params, current, pageSize}).then((resp: any) => {
|
||||
let { current, pageSize } = data.page;
|
||||
let params = data.params;
|
||||
ApiGet(`/manage/collect/record/list`, { ...params, current, pageSize }).then(
|
||||
(resp: any) => {
|
||||
if (resp.code === 0) {
|
||||
resp.data.list.map((item: any) => {
|
||||
// 对数据进行格式化处理
|
||||
item.timeFormat = fmt.dateFormat(new Date(item.UpdatedAt).getTime())
|
||||
return item
|
||||
})
|
||||
data.records = resp.data.list
|
||||
data.page = resp.data.params.paging
|
||||
item.timeFormat = fmt.dateFormat(new Date(item.UpdatedAt).getTime());
|
||||
return item;
|
||||
});
|
||||
data.records = resp.data.list;
|
||||
data.page = resp.data.params.paging;
|
||||
|
||||
// 初始化select参数
|
||||
data.options = resp.data.options;
|
||||
} else {
|
||||
ElMessage.error({message: resp.msg})
|
||||
ElMessage.error({ message: resp.msg });
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
// 处理筛选参数, 获取满足条件的记录
|
||||
const filterRecord = () => {
|
||||
if (data.dateGroup && data.dateGroup.length == 2) {
|
||||
data.params.beginTime = data.dateGroup[0];
|
||||
data.params.endTime = data.dateGroup[1];
|
||||
} else {
|
||||
data.params.beginTime = "";
|
||||
data.params.endTime = "";
|
||||
}
|
||||
getRecords();
|
||||
};
|
||||
|
||||
// 恢复采集, 对已采集失败的记录进行重试操作
|
||||
const collectRecover = (id:number)=>{
|
||||
ApiGet(`/manage/collect/record/retry`, {id:id}).then((resp: any) => {
|
||||
if(resp.code === 0){
|
||||
ElMessage.success({message: resp.msg})
|
||||
} else{
|
||||
ElMessage.error({message: resp.msg})
|
||||
const collectRecover = (id: number) => {
|
||||
ApiGet(`/manage/collect/record/retry`, { id: id }).then((resp: any) => {
|
||||
if (resp.code === 0) {
|
||||
ElMessage.success({ message: resp.msg });
|
||||
} else {
|
||||
ElMessage.error({ message: resp.msg });
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// 删除当前记录, 记录删除貌似不合理, 留以待定
|
||||
const delRecord = (id: number) => {};
|
||||
|
||||
// collectRecoverAll 对目前记录的所有未处理记录进行重新采集
|
||||
const collectRecoverAll = () => {
|
||||
ElMessageBox.confirm("是否对所有失效记录进行重新采集?", "采集失败记录处理", {
|
||||
confirmButtonText: "执行",
|
||||
cancelButtonText: "取消",
|
||||
type: "warning",
|
||||
center: true,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
// 删除当前记录, 貌似不合理
|
||||
const delRecord = (id:number)=>{
|
||||
|
||||
}
|
||||
.then(() => {
|
||||
ApiGet(`/manage/collect/record/retry/all`).then((resp: any) => {
|
||||
if (resp.code === 0) {
|
||||
ElMessage.success({ message: resp.msg });
|
||||
} else {
|
||||
ElMessage.error({ message: resp.msg });
|
||||
}
|
||||
});
|
||||
})
|
||||
.catch(() => {
|
||||
ElMessage({ type: "warning", message: "采集恢复操作已取消!!!" });
|
||||
});
|
||||
};
|
||||
|
||||
// cleanDoneRecord 清除已处理的记录
|
||||
const cleanDoneRecord = () => {
|
||||
ElMessageBox.confirm("是否清除所有已处理的记录?", "记录清除", {
|
||||
confirmButtonText: "执行",
|
||||
cancelButtonText: "取消",
|
||||
type: "warning",
|
||||
center: true,
|
||||
})
|
||||
.then(() => {
|
||||
ApiGet(`/manage/collect/record/clear/done`).then((resp: any) => {
|
||||
if (resp.code === 0) {
|
||||
ElMessage.success({ message: resp.msg });
|
||||
} else {
|
||||
ElMessage.error({ message: resp.msg });
|
||||
}
|
||||
});
|
||||
})
|
||||
.catch(() => {
|
||||
ElMessage({ type: "warning", message: "记录清除已取消!!!" });
|
||||
});
|
||||
};
|
||||
|
||||
// 清除所有失败采集记录
|
||||
const clearAllRecord = ()=>{
|
||||
|
||||
const cleanAllRecord = () => {
|
||||
ElMessageBox.confirm("是否清除所有记录?", "记录清除", {
|
||||
confirmButtonText: "执行",
|
||||
cancelButtonText: "取消",
|
||||
type: "warning",
|
||||
center: true,
|
||||
}).then(() => {
|
||||
ApiGet(`/manage/collect/record/clear/all`).then((resp: any) => {
|
||||
if (resp.code === 0) {
|
||||
ElMessage.success({ message: resp.msg });
|
||||
} else {
|
||||
ElMessage.error({ message: resp.msg });
|
||||
}
|
||||
})
|
||||
}).catch(() => {
|
||||
ElMessage({ type: "warning", message: "记录清除已取消!!!" })
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
onMounted(() => {
|
||||
// 获取分页数据
|
||||
getRecords()
|
||||
console.log(data)
|
||||
})
|
||||
|
||||
|
||||
getRecords();
|
||||
console.log(data);
|
||||
});
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
/*头部检索表单*/
|
||||
.params_form {
|
||||
background: var(--bg-light);
|
||||
margin-bottom: 20px;
|
||||
padding: 10px 20px;
|
||||
}
|
||||
|
||||
.cus_form {
|
||||
width: 100%;
|
||||
flex-flow: wrap;
|
||||
display: flex;
|
||||
justify-content: start;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
:deep(.el-form-item) {
|
||||
width: calc(16% - 12px);
|
||||
margin: 10px 6px;
|
||||
}
|
||||
|
||||
.content {
|
||||
border: 1px solid #9b49e733;
|
||||
@@ -159,6 +386,8 @@ onMounted(() => {
|
||||
max-width: 100%;
|
||||
text-align: center;
|
||||
padding-right: 50px;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
}
|
||||
|
||||
:deep(.el-pagination) {
|
||||
@@ -177,4 +406,8 @@ onMounted(() => {
|
||||
--el-pagination-button-bg-color: var(--btn-bg-linght);
|
||||
border: 1px solid var(--border-gray-color);
|
||||
}
|
||||
|
||||
.cus_util {
|
||||
border: none;
|
||||
}
|
||||
</style>
|
||||
@@ -11,7 +11,7 @@
|
||||
<el-table-column prop="remark" label="任务描述" />
|
||||
<el-table-column prop="model" align="center" label="任务类型">
|
||||
<template #default="scope">
|
||||
<el-tag disable-transitions>{{ scope.row.model == 0 ? '自动更新':'自定义任务'}}</el-tag>
|
||||
<el-tag disable-transitions>{{ scope.row.model == 0 ? '自动更新':scope.row.model == 0 ?'自定义任务':'采集重试'}}</el-tag>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="state" align="center" label="是否启用">
|
||||
@@ -58,6 +58,9 @@
|
||||
<el-tooltip class="box-item" effect="dark" content="只执行指定站点的采集任务" placement="top">
|
||||
<el-radio :label="1">自定义更新</el-radio>
|
||||
</el-tooltip>
|
||||
<el-tooltip class="box-item" effect="dark" content="失败采集重试处理" placement="top">
|
||||
<el-radio :label="2">采集重试</el-radio>
|
||||
</el-tooltip>
|
||||
</el-radio-group>
|
||||
</el-form-item>
|
||||
<el-form-item v-if="form.add.model == 1" label="资源绑定">
|
||||
@@ -65,7 +68,7 @@
|
||||
<el-option v-for="item in form.options" :key="item.id" :label="item.name" :value="item.id"/>
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
<el-form-item label="采集时长">
|
||||
<el-form-item v-if="form.add.model != 2" label="采集时长">
|
||||
<el-tooltip class="box-item" effect="dark" content="采集最近x小时更新的影片,负数则默认采集所有资源" placement="top">
|
||||
<el-input-number v-model="form.add.time" :step="1" step-strictly />
|
||||
</el-tooltip>
|
||||
@@ -94,14 +97,14 @@
|
||||
<el-tag disable-transitions>{{ form.edit.spec }}</el-tag>
|
||||
</el-form-item>
|
||||
<el-form-item label="任务类型">
|
||||
<el-tag disable-transitions>{{ form.edit.model == 0?'自动更新':'自定义更新' }}</el-tag>
|
||||
<el-tag disable-transitions>{{ form.edit.model == 0?'自动更新':form.edit.model == 1?'自定义更新':'采集重试' }}</el-tag>
|
||||
</el-form-item>
|
||||
<el-form-item v-if="form.edit.model == 1" label="资源绑定">
|
||||
<el-select v-model="form.edit.ids" multiple collapse-tags collapse-tags-tooltip placeholder="Select" style="width: 240px">
|
||||
<el-option v-for="item in form.options" :key="item.id" :label="item.name" :value="item.id"/>
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
<el-form-item label="采集时长">
|
||||
<el-form-item v-if="form.edit.model != 2" label="采集时长">
|
||||
<el-tooltip class="box-item" effect="dark" content="采集最近x小时更新的影片,负数则默认采集所有资源" placement="top">
|
||||
<el-input-number v-model="form.edit.time" :step="1" step-strictly />
|
||||
</el-tooltip>
|
||||
|
||||
@@ -70,7 +70,7 @@ const (
|
||||
// DefaultUpdateSpec 每20分钟执行一次
|
||||
DefaultUpdateSpec = "0 */20 * * * ?"
|
||||
// EveryWeekSpec 每周日凌晨4点更新一次
|
||||
EveryWeekSpec = "0 0 4 * * 7"
|
||||
EveryWeekSpec = "0 0 4 * * 0"
|
||||
// DefaultUpdateTime 每次采集最近 3 小时内更新的影片
|
||||
DefaultUpdateTime = 3
|
||||
)
|
||||
@@ -93,7 +93,7 @@ const (
|
||||
//mysql服务配置信息 root:root 设置mysql账户的用户名和密码
|
||||
|
||||
MysqlDsn = "root:root@(192.168.20.5:3306)/FilmSite?charset=utf8mb4&parseTime=True&loc=Local"
|
||||
//MysqlDsn = "root:MuBai0916$@(1.94.30.26:3610)/FilmSite?charset=utf8mb4&parseTime=True&loc=Local"
|
||||
//MysqlDsn = "root:MuBai0916$@(113.44.5.201:3610)/FilmSite?charset=utf8mb4&parseTime=True&loc=Local"
|
||||
|
||||
// MysqlDsn docker compose 环境下的链接信息 mysql:3306 为 docker compose 中 mysql服务对应的网络名称和端口
|
||||
//MysqlDsn = "root:root@(mysql:3306)/FilmSite?charset=utf8mb4&parseTime=True&loc=Local"
|
||||
@@ -104,8 +104,10 @@ const (
|
||||
RedisPassword redis访问密码
|
||||
RedisDBNo 使用第几号库
|
||||
*/
|
||||
//RedisAddr = `1.94.30.26:3620`
|
||||
//RedisAddr = `113.44.5.201:3620`
|
||||
//RedisPassword = `MuBai0916$`
|
||||
//RedisDBNo = 0
|
||||
|
||||
RedisAddr = `192.168.20.5:6379`
|
||||
RedisPassword = `root`
|
||||
RedisDBNo = 0
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"server/model/system"
|
||||
"server/plugin/spider"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ------------------------------------------------------ 影视采集 ------------------------------------------------------
|
||||
@@ -199,10 +200,30 @@ func FailureRecordList(c *gin.Context) {
|
||||
var err error
|
||||
// 获取筛选条件
|
||||
params.OriginId = c.DefaultQuery("originId", "")
|
||||
params.CollectType, err = strconv.Atoi(c.DefaultQuery("collectType", "-1"))
|
||||
//params.CollectType, err = strconv.Atoi(c.DefaultQuery("collectType", "-1"))
|
||||
params.Hour, err = strconv.Atoi(c.DefaultQuery("hour", "0"))
|
||||
params.Status, err = strconv.Atoi(c.DefaultQuery("status", "-1"))
|
||||
|
||||
// 处理时间参数
|
||||
begin := c.DefaultQuery("beginTime", "")
|
||||
if begin != "" {
|
||||
beginTime, e := time.ParseInLocation(time.DateTime, begin, time.Local)
|
||||
if e != nil {
|
||||
system.Failed("影片分页数据获取失败, 请求参数异常", c)
|
||||
return
|
||||
}
|
||||
params.BeginTime = beginTime
|
||||
}
|
||||
end := c.DefaultQuery("endTime", "")
|
||||
if end != "" {
|
||||
endTime, e := time.ParseInLocation(time.DateTime, end, time.Local)
|
||||
if e != nil {
|
||||
system.Failed("影片分页数据获取失败, 请求参数异常", c)
|
||||
return
|
||||
}
|
||||
params.EndTime = endTime
|
||||
}
|
||||
|
||||
// 分页参数
|
||||
params.Paging.Current, err = strconv.Atoi(c.DefaultQuery("current", "1"))
|
||||
params.Paging.PageSize, err = strconv.Atoi(c.DefaultQuery("pageSize", "10"))
|
||||
@@ -215,12 +236,15 @@ func FailureRecordList(c *gin.Context) {
|
||||
params.Paging.PageSize = 10
|
||||
}
|
||||
|
||||
// 条件筛选select选项参数
|
||||
options := logic.CollectL.GetRecordOptions()
|
||||
|
||||
// 获取满足条件的分页数据
|
||||
list := logic.CollectL.GetRecordList(params)
|
||||
system.Success(gin.H{"params": params, "list": list}, "影片分页信息获取成功", c)
|
||||
system.Success(gin.H{"params": params, "list": list, "options": options}, "影片分页信息获取成功", c)
|
||||
}
|
||||
|
||||
// CollectRecover 对失败的采集进行
|
||||
// CollectRecover 对失败的采集进行处理
|
||||
func CollectRecover(c *gin.Context) {
|
||||
// 获取记录id
|
||||
id, err := strconv.Atoi(c.DefaultQuery("id", "0"))
|
||||
@@ -236,3 +260,24 @@ func CollectRecover(c *gin.Context) {
|
||||
}
|
||||
system.SuccessOnlyMsg("采集重试已开启, 请勿重复操作", c)
|
||||
}
|
||||
|
||||
// CollectRecoverAll 恢复采集-全量
|
||||
func CollectRecoverAll(c *gin.Context) {
|
||||
// 重新采集表中所有失败记录
|
||||
logic.CollectL.RecoverAll()
|
||||
system.SuccessOnlyMsg("恢复任务已成功开启!!!", c)
|
||||
}
|
||||
|
||||
// ClearDoneRecord 清理已处理的记录
|
||||
func ClearDoneRecord(c *gin.Context) {
|
||||
// 删除表中已处理完成的记录
|
||||
logic.CollectL.ClearDoneRecord()
|
||||
system.SuccessOnlyMsg("处理完成的记录信息已删除!!!", c)
|
||||
}
|
||||
|
||||
// ClearAllRecord 删除所有记录
|
||||
func ClearAllRecord(c *gin.Context) {
|
||||
// 截断失败采集记录表
|
||||
logic.CollectL.ClearAllRecord()
|
||||
system.SuccessOnlyMsg("采集异常记录信息已清空!!!", c)
|
||||
}
|
||||
|
||||
@@ -140,17 +140,34 @@ func validTaskInfo(t system.FilmCollectTask) error {
|
||||
|
||||
// 任务添加参数校验
|
||||
func validTaskAddVo(vo system.FilmCronVo) error {
|
||||
if vo.Model != 0 && vo.Model != 1 && vo.Model != 2 {
|
||||
return errors.New("参数校验失败, 未定义的任务类型")
|
||||
}
|
||||
switch vo.Model {
|
||||
case 0:
|
||||
if vo.Time == 0 {
|
||||
return errors.New("参数校验失败, 采集时长不能为零值")
|
||||
}
|
||||
case 1:
|
||||
if vo.Time == 0 {
|
||||
return errors.New("参数校验失败, 采集时长不能为零值")
|
||||
}
|
||||
if vo.Ids == nil || len(vo.Ids) <= 0 {
|
||||
return errors.New("参数校验失败, 自定义更新未绑定任何资源站点")
|
||||
}
|
||||
case 2:
|
||||
break
|
||||
default:
|
||||
return errors.New("参数校验失败, 未定义的任务类型")
|
||||
}
|
||||
//if vo.Model != 0 && vo.Model != 1 && vo.Model != 2 {
|
||||
// return errors.New("参数校验失败, 未定义的任务类型")
|
||||
//}
|
||||
//if vo.Time == 0 {
|
||||
// return errors.New("参数校验失败, 采集时长不能为零值")
|
||||
//}
|
||||
if err := spider.ValidSpec(vo.Spec); err != nil {
|
||||
return errors.New(fmt.Sprint("参数校验失败 cron表达式校验失败: ", err.Error()))
|
||||
}
|
||||
if vo.Model == 1 && (vo.Ids == nil || len(vo.Ids) <= 0) {
|
||||
return errors.New("参数校验失败, 自定义更新未绑定任何资源站点")
|
||||
}
|
||||
//if vo.Model == 1 && (vo.Ids == nil || len(vo.Ids) <= 0) {
|
||||
// return errors.New("参数校验失败, 自定义更新未绑定任何资源站点")
|
||||
//}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -56,6 +56,21 @@ func (cl *CollectLogic) GetRecordList(params system.RecordRequestVo) []system.Fa
|
||||
return system.FailureRecordList(params)
|
||||
}
|
||||
|
||||
// GetRecordOptions 获取采集记录筛选参数
|
||||
func (cl *CollectLogic) GetRecordOptions() system.OptionGroup {
|
||||
var options = make(system.OptionGroup)
|
||||
// 获取筛选参数, 采集源(ID:name) | 采集类型 | 状态
|
||||
options["collectType"] = []system.Option{{"全部", -1}, {"影片详情", 0}, {"文章", 1}, {"演员", 2}, {"角色", 3}, {"网站", 4}}
|
||||
options["status"] = []system.Option{{"全部", -1}, {"待重试", 1}, {"已处理", 0}}
|
||||
// 获取全部采集站
|
||||
var originOptions = []system.Option{{"全部", ""}}
|
||||
for _, v := range system.GetCollectSourceList() {
|
||||
originOptions = append(originOptions, system.Option{Name: v.Name, Value: v.Id})
|
||||
}
|
||||
options["origin"] = originOptions
|
||||
return options
|
||||
}
|
||||
|
||||
// CollectRecover 恢复采集
|
||||
func (cl *CollectLogic) CollectRecover(id int) error {
|
||||
// 通过ID获取完整的失败记录信息
|
||||
@@ -69,3 +84,23 @@ func (cl *CollectLogic) CollectRecover(id int) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RecoverAll 恢复重新对所有失效记录进行重新采集处理
|
||||
func (cl *CollectLogic) RecoverAll() {
|
||||
// 是否进行身份验证, 暂定无需处理
|
||||
|
||||
// 对数据表中的所有待处理记录进行恢复采集
|
||||
go spider.FullRecoverSpider()
|
||||
}
|
||||
|
||||
// ClearDoneRecord 清除已处理完成的记录信息 (将记录表中已经完成处理的记录删除)
|
||||
func (cl *CollectLogic) ClearDoneRecord() {
|
||||
// <逻辑删除 or 真实删除> 为避免ID中断暂定逻辑删除
|
||||
system.DelDoneRecord()
|
||||
}
|
||||
|
||||
// ClearAllRecord 清除所有记录信息 (直接对记录表直接进行截断处理)
|
||||
func (cl *CollectLogic) ClearAllRecord() {
|
||||
// 重置记录表状态, 删除所有数据并将自增ID归零
|
||||
system.TruncateRecordTable()
|
||||
}
|
||||
|
||||
@@ -44,7 +44,7 @@ func (cl *CronLogic) AddFilmCrontab(cv system.FilmCronVo) error {
|
||||
cid, err := spider.AddFilmRecoverCron(task.Spec)
|
||||
// 如果任务添加失败则直接返回错误信息
|
||||
if err != nil {
|
||||
return errors.New(fmt.Sprint("影视更新定时任务添加失败: ", err.Error()))
|
||||
return errors.New(fmt.Sprint("失败采集处理定时任务添加失败: ", err.Error()))
|
||||
}
|
||||
// 将定时任务Id记录到Task中
|
||||
task.Cid = cid
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package system
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"gorm.io/gorm"
|
||||
"log"
|
||||
"server/config"
|
||||
@@ -50,15 +51,18 @@ func FailureRecordList(vo RecordRequestVo) []FailureRecord {
|
||||
if vo.OriginId != "" {
|
||||
qw.Where("origin_id = ?", vo.OriginId)
|
||||
}
|
||||
if !vo.BeginTime.IsZero() && !vo.EndTime.IsZero() {
|
||||
qw.Where("created_at BETWEEN ? AND ? ", vo.BeginTime, vo.EndTime)
|
||||
}
|
||||
//if vo.CollectType >= 0 {
|
||||
// qw.Where("collect_type = ?", vo.CollectType)
|
||||
//}
|
||||
//if vo.Hour != 0 {
|
||||
// qw.Where("hour = ?", vo.Hour)
|
||||
//}
|
||||
//if vo.Status >= 0 {
|
||||
// qw.Where("status = ?", vo.Status)
|
||||
//}
|
||||
if vo.Status >= 0 {
|
||||
qw.Where("status = ?", vo.Status)
|
||||
}
|
||||
|
||||
// 获取分页数据
|
||||
GetPage(qw, vo.Paging)
|
||||
@@ -114,3 +118,19 @@ func RetryRecord(id uint, status int64) error {
|
||||
return db.Mdb.Model(&FailureRecord{}).Where("update_at > ?", fr.UpdatedAt).Update("status", 0).Error
|
||||
|
||||
}
|
||||
|
||||
// DelDoneRecord 删除已处理的记录信息 -- 逻辑删除
|
||||
func DelDoneRecord() {
|
||||
if err := db.Mdb.Where("status = ?", 0).Delete(&FailureRecord{}).Error; err != nil {
|
||||
log.Println("Delete failure record failed:", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TruncateRecordTable 截断 record table
|
||||
func TruncateRecordTable() {
|
||||
var s FailureRecord
|
||||
err := db.Mdb.Exec(fmt.Sprintf("TRUNCATE Table %s", s.TableName())).Error
|
||||
if err != nil {
|
||||
log.Println("TRUNCATE TABLE Error: ", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,19 +73,19 @@ func FilmZero() {
|
||||
db.Rdb.Del(db.Cxt, db.Rdb.Keys(db.Cxt, "OriginalResource*").Val()...)
|
||||
db.Rdb.Del(db.Cxt, db.Rdb.Keys(db.Cxt, "Search*").Val()...)
|
||||
// 删除mysql中留存的检索表
|
||||
var s *SearchInfo
|
||||
var s SearchInfo
|
||||
//db.Mdb.Exec(fmt.Sprintf(`drop table if exists %s`, s.TableName()))
|
||||
// 截断数据表 truncate table users
|
||||
if ExistSearchTable() {
|
||||
db.Mdb.Exec(fmt.Sprintf(`TRUNCATE table %s`, s.TableName()))
|
||||
db.Mdb.Exec(fmt.Sprintf("TRUNCATE table %s", s.TableName()))
|
||||
}
|
||||
}
|
||||
|
||||
// ResetSearchTable 重置Search表
|
||||
func ResetSearchTable() {
|
||||
// 删除 Search 表
|
||||
var s *SearchInfo
|
||||
db.Mdb.Exec(fmt.Sprintf(`drop table if exists %s`, s.TableName()))
|
||||
var s SearchInfo
|
||||
db.Mdb.Exec(fmt.Sprintf("drop table if exists %s", s.TableName()))
|
||||
// 重新创建 Search 表
|
||||
CreateSearchTable()
|
||||
}
|
||||
@@ -230,6 +230,7 @@ func CreateSearchTable() {
|
||||
}
|
||||
}
|
||||
|
||||
// ExistSearchTable 是否存在Search Table
|
||||
func ExistSearchTable() bool {
|
||||
// 1. 判断表中是否存在当前表
|
||||
return db.Mdb.Migrator().HasTable(&SearchInfo{})
|
||||
@@ -237,7 +238,7 @@ func ExistSearchTable() bool {
|
||||
|
||||
// AddSearchIndex search表中数据保存完毕后 将常用字段添加索引提高查询效率
|
||||
func AddSearchIndex() {
|
||||
var s *SearchInfo
|
||||
var s SearchInfo
|
||||
tableName := s.TableName()
|
||||
// 添加索引
|
||||
db.Mdb.Exec(fmt.Sprintf("CREATE UNIQUE INDEX idx_mid ON %s (mid)", tableName))
|
||||
@@ -331,8 +332,8 @@ func ExistSearchInfo(mid int64) bool {
|
||||
|
||||
// TunCateSearchTable 截断SearchInfo数据表
|
||||
func TunCateSearchTable() {
|
||||
var searchInfo *SearchInfo
|
||||
err := db.Mdb.Exec(fmt.Sprint("TRUNCATE TABLE ", searchInfo.TableName())).Error
|
||||
var searchInfo SearchInfo
|
||||
err := db.Mdb.Exec(fmt.Sprintf("TRUNCATE TABLE %s", searchInfo.TableName())).Error
|
||||
if err != nil {
|
||||
log.Println("TRUNCATE TABLE Error: ", err)
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@ func CreateUserTable() {
|
||||
// 如果不存在则创建表 并设置自增ID初始值为10000
|
||||
if !ExistUserTable() {
|
||||
err := db.Mdb.AutoMigrate(u)
|
||||
db.Mdb.Exec(fmt.Sprintf("alter table %s auto_Increment=%d", u.TableName(), config.UserIdInitialVal))
|
||||
db.Mdb.Exec(fmt.Sprintf("alter table %s auto_Increment = %s", u.TableName(), config.UserIdInitialVal))
|
||||
if err != nil {
|
||||
log.Println("Create Table SearchInfo Failed: ", err)
|
||||
}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package system
|
||||
|
||||
import "time"
|
||||
|
||||
// SearchTagsVO 搜索标签请求参数
|
||||
type SearchTagsVO struct {
|
||||
Pid int64 `json:"pid"`
|
||||
@@ -34,6 +36,12 @@ type FilmTaskOptions struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type Option struct {
|
||||
Name string `json:"name"`
|
||||
Value any `json:"value"`
|
||||
}
|
||||
type OptionGroup map[string][]Option
|
||||
|
||||
// CollectParams 数据采集所需要的参数
|
||||
type CollectParams struct {
|
||||
Id string `json:"id"` // 资源站id
|
||||
@@ -120,5 +128,7 @@ type RecordRequestVo struct {
|
||||
CollectType int `json:"collectType"` // 采集类型
|
||||
Hour int `json:"hour"` // 采集时长
|
||||
Status int `json:"status"` // 状态
|
||||
BeginTime time.Time `json:"beginTime"` // 起始时间
|
||||
EndTime time.Time `json:"endTime"` // 结束时间
|
||||
Paging *Page `json:"paging"` // 分页参数
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package db
|
||||
import (
|
||||
"gorm.io/driver/mysql"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
"gorm.io/gorm/schema"
|
||||
"server/config"
|
||||
)
|
||||
@@ -25,7 +24,7 @@ func InitMysql() (err error) {
|
||||
SingularTable: true, //是否使用 结构体名称作为表名 (关闭自动变复数)
|
||||
//NameReplacer: strings.NewReplacer("spider_", ""), // 替表名和字段中的 Me 为 空
|
||||
},
|
||||
Logger: logger.Default.LogMode(logger.Info), //设置日志级别为Info
|
||||
//Logger: logger.Default.LogMode(logger.Info), //设置日志级别为Info
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
@@ -336,7 +336,7 @@ func FullRecoverSpider() {
|
||||
case fr.Hour > 0 && fr.Hour < 4320:
|
||||
// 将此记录之后的所有同类采集记录变更为已重试
|
||||
system.ChangeRecord(&fr, 0)
|
||||
// 如果采集的内容是 7~15 天之内更新的内容,则采集此记录之后的所有更新内容
|
||||
// 如果采集的内容是 0~180 天之内更新的内容,则采集此记录之后的所有更新内容
|
||||
// 获取采集参数h, 采集时长变更为 原采集时长 + 采集记录距现在的时长
|
||||
h := fr.Hour + int(math.Ceil(time.Since(fr.CreatedAt).Hours()))
|
||||
// 对当前所有已启用的站点 更新最新 h 小时的内容
|
||||
@@ -344,7 +344,7 @@ func FullRecoverSpider() {
|
||||
case fr.Hour < 0, fr.Hour > 4320:
|
||||
// 将此记录状态修改为已重试
|
||||
system.ChangeRecord(&fr, 0)
|
||||
// 如果采集的是 最近180天内更新的内容 或全部内容, 则只对当前一条记录进行二次采集
|
||||
// 如果采集的是 180天之前更新的内容 或全部内容, 则只对当前一条记录进行二次采集
|
||||
s := system.FindCollectSourceById(fr.OriginId)
|
||||
collectFilm(s, fr.Hour, fr.PageNumber)
|
||||
default:
|
||||
|
||||
@@ -18,7 +18,7 @@ func CreateCron() *cron.Cron {
|
||||
return cron.New(cron.WithSeconds())
|
||||
}
|
||||
|
||||
// AddFilmUpdateCron 添加影片更新定时任务
|
||||
// AddFilmUpdateCron 添加 指定站点的影片更新定时任务
|
||||
func AddFilmUpdateCron(id, spec string) (cron.EntryID, error) {
|
||||
// 校验 spec 表达式的有效性
|
||||
if err := ValidSpec(spec); err != nil {
|
||||
@@ -40,7 +40,7 @@ func AddFilmUpdateCron(id, spec string) (cron.EntryID, error) {
|
||||
})
|
||||
}
|
||||
|
||||
// AddAutoUpdateCron 自动更新定时任务
|
||||
// AddAutoUpdateCron 添加 所有已启用站点的影片更新定时任务
|
||||
func AddAutoUpdateCron(id, spec string) (cron.EntryID, error) {
|
||||
// 校验 spec 表达式的有效性
|
||||
if err := ValidSpec(spec); err != nil {
|
||||
|
||||
@@ -75,6 +75,10 @@ func SetupRouter() *gin.Engine {
|
||||
|
||||
collect.GET(`/record/list`, controller.FailureRecordList)
|
||||
collect.GET(`/record/retry`, controller.CollectRecover)
|
||||
collect.GET(`/record/retry/all`, controller.CollectRecoverAll)
|
||||
collect.GET(`/record/clear/done`, controller.ClearDoneRecord)
|
||||
collect.GET(`/record/clear/all`, controller.ClearAllRecord)
|
||||
|
||||
}
|
||||
|
||||
// 定时任务相关
|
||||
|
||||
Reference in New Issue
Block a user