fnvtk/cunkebao_v3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

数据中心同步

Browse Source
This commit is contained in:
乘风
2026-01-05 10:16:20 +08:00
parent 0457528dd0
commit ba0ebcf273
98 changed files with 28583 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

309
Moncter/app/service/DataSource/Adapter/MongoDBAdapter.php Normal file
View File

@@ -0,0 +1,309 @@
<?php
namespace app\service\DataSource\Adapter;
use app\service\DataSource\DataSourceAdapterInterface;
use app\utils\LoggerHelper;
use MongoDB\Client;
use MongoDB\Driver\Exception\Exception as MongoDBException;
/**
* MongoDB 数据源适配器
*
* 职责：
* - 封装 MongoDB 数据库连接和查询操作
* - 实现 DataSourceAdapterInterface 接口
*/
class MongoDBAdapter implements DataSourceAdapterInterface
{
private ?Client $client = null;
private ?\MongoDB\Database $database = null;
private string $type = 'mongodb';
private string $databaseName = '';
/**
* 建立数据库连接
*
* @param array<string, mixed> $config 数据源配置
* @return bool 是否连接成功
*/
public function connect(array $config): bool
{
try {
$host = $config['host'] ?? '127.0.0.1';
$port = (int)($config['port'] ?? 27017);
$this->databaseName = $config['database'] ?? '';
$username = $config['username'] ?? '';
$password = $config['password'] ?? '';
$authSource = $config['auth_source'] ?? $this->databaseName;
// 构建 DSN
$dsn = "mongodb://";
if (!empty($username) && !empty($password)) {
$dsn .= urlencode($username) . ':' . urlencode($password) . '@';
}
$dsn .= "{$host}:{$port}";
if (!empty($this->databaseName)) {
$dsn .= "/{$this->databaseName}";
}
if (!empty($authSource)) {
$dsn .= "?authSource=" . urlencode($authSource);
}
// MongoDB 连接选项
$options = [];
if (isset($config['options'])) {
$options = array_filter($config['options'], function ($value) {
return $value !== '' && $value !== null;
});
}
// 设置超时选项
if (!isset($options['connectTimeoutMS'])) {
$options['connectTimeoutMS'] = ($config['timeout'] ?? 10) * 1000;
}
if (!isset($options['socketTimeoutMS'])) {
$options['socketTimeoutMS'] = ($config['timeout'] ?? 10) * 1000;
}
$this->client = new Client($dsn, $options);
// 选择数据库
if (!empty($this->databaseName)) {
$this->database = $this->client->selectDatabase($this->databaseName);
}
// 测试连接
$this->client->getManager()->selectServer();
LoggerHelper::logBusiness('mongodb_adapter_connected', [
'host' => $host,
'port' => $port,
'database' => $this->databaseName,
]);
return true;
} catch (MongoDBException $e) {
LoggerHelper::logError($e, [
'component' => 'MongoDBAdapter',
'action' => 'connect',
'config' => array_merge($config, ['password' => '***']), // 隐藏密码
]);
return false;
}
}
/**
* 关闭数据库连接
*
* @return void
*/
public function disconnect(): void
{
if ($this->client !== null) {
$this->client = null;
$this->database = null;
LoggerHelper::logBusiness('mongodb_adapter_disconnected', []);
}
}
/**
* 测试连接是否有效
*
* @return bool 连接是否有效
*/
public function isConnected(): bool
{
if ($this->client === null) {
return false;
}
try {
// 执行 ping 命令测试连接
$adminDb = $this->client->selectDatabase('admin');
$adminDb->command(['ping' => 1]);
return true;
} catch (MongoDBException $e) {
LoggerHelper::logError($e, [
'component' => 'MongoDBAdapter',
'action' => 'isConnected',
]);
return false;
}
}
/**
* 执行查询（返回多条记录）
*
* 注意：对于 MongoDB，$sql 参数表示集合名称，$params 是一个包含 'filter' 和 'options' 的数组
*
* @param string $sql 集合名称（MongoDB 中相当于表名）
* @param array<string, mixed> $params 查询参数，格式：['filter' => [...], 'options' => [...]]
* @return array<array<string, mixed>> 查询结果数组
*/
public function query(string $sql, array $params = []): array
{
if ($this->database === null) {
throw new \RuntimeException('数据库连接未建立或未选择数据库');
}
try {
$collection = $sql; // $sql 参数在 MongoDB 中表示集合名
$filter = $params['filter'] ?? [];
$options = $params['options'] ?? [];
$cursor = $this->database->selectCollection($collection)->find($filter, $options);
$results = [];
foreach ($cursor as $document) {
$results[] = $this->convertMongoDocumentToArray($document);
}
LoggerHelper::logBusiness('mongodb_query_executed', [
'collection' => $collection,
'filter' => $filter,
'result_count' => count($results),
]);
return $results;
} catch (MongoDBException $e) {
LoggerHelper::logError($e, [
'component' => 'MongoDBAdapter',
'action' => 'query',
'collection' => $sql,
'params' => $params,
]);
throw $e;
}
}
/**
* 执行查询（返回单条记录）
*
* 注意：对于 MongoDB，$sql 参数表示集合名称，$params 是一个包含 'filter' 和 'options' 的数组
*
* @param string $sql 集合名称
* @param array<string, mixed> $params 查询参数，格式：['filter' => [...], 'options' => [...]]
* @return array<string, mixed>|null 查询结果（单条记录）或 null
*/
public function queryOne(string $sql, array $params = []): ?array
{
if ($this->database === null) {
throw new \RuntimeException('数据库连接未建立或未选择数据库');
}
try {
$collection = $sql; // $sql 参数在 MongoDB 中表示集合名
$filter = $params['filter'] ?? [];
$options = $params['options'] ?? [];
$document = $this->database->selectCollection($collection)->findOne($filter, $options);
if ($document === null) {
return null;
}
LoggerHelper::logBusiness('mongodb_query_one_executed', [
'collection' => $collection,
'filter' => $filter,
'has_result' => true,
]);
return $this->convertMongoDocumentToArray($document);
} catch (MongoDBException $e) {
LoggerHelper::logError($e, [
'component' => 'MongoDBAdapter',
'action' => 'queryOne',
'collection' => $sql,
'params' => $params,
]);
throw $e;
}
}
/**
* 批量查询（分页查询，用于大数据量场景）
*
* 注意：对于 MongoDB，$sql 参数表示集合名称，$params 是一个包含 'filter' 和 'options' 的数组
*
* @param string $sql 集合名称
* @param array<string, mixed> $params 查询参数，格式：['filter' => [...], 'options' => [...]]
* @param int $offset 偏移量
* @param int $limit 每页数量
* @return array<array<string, mixed>> 查询结果数组
*/
public function queryBatch(string $sql, array $params = [], int $offset = 0, int $limit = 1000): array
{
if ($this->database === null) {
throw new \RuntimeException('数据库连接未建立或未选择数据库');
}
try {
$collection = $sql; // $sql 参数在 MongoDB 中表示集合名
$filter = $params['filter'] ?? [];
$options = $params['options'] ?? [];
// 设置分页选项
$options['skip'] = $offset;
$options['limit'] = $limit;
$cursor = $this->database->selectCollection($collection)->find($filter, $options);
$results = [];
foreach ($cursor as $document) {
$results[] = $this->convertMongoDocumentToArray($document);
}
LoggerHelper::logBusiness('mongodb_query_batch_executed', [
'collection' => $collection,
'offset' => $offset,
'limit' => $limit,
'result_count' => count($results),
]);
return $results;
} catch (MongoDBException $e) {
LoggerHelper::logError($e, [
'component' => 'MongoDBAdapter',
'action' => 'queryBatch',
'collection' => $sql,
'params' => $params,
'offset' => $offset,
'limit' => $limit,
]);
throw $e;
}
}
/**
* 获取数据源类型
*
* @return string 数据源类型
*/
public function getType(): string
{
return $this->type;
}
/**
* 将 MongoDB 文档转换为数组
*
* @param mixed $document MongoDB 文档对象
* @return array<string, mixed> 数组格式的数据
*/
private function convertMongoDocumentToArray($document): array
{
if (is_array($document)) {
return $document;
}
// MongoDB\BSON\Document 或 MongoDB\Model\BSONDocument
if (method_exists($document, 'toArray')) {
return $document->toArray();
}
// 转换为数组
return json_decode(json_encode($document), true) ?? [];
}
}

234
Moncter/app/service/DataSource/Adapter/MySQLAdapter.php Normal file
View File

@@ -0,0 +1,234 @@
<?php
namespace app\service\DataSource\Adapter;
use app\service\DataSource\DataSourceAdapterInterface;
use app\utils\LoggerHelper;
use PDO;
use PDOException;
/**
* MySQL 数据源适配器
*
* 职责：
* - 封装 MySQL 数据库连接和查询操作
* - 实现 DataSourceAdapterInterface 接口
*/
class MySQLAdapter implements DataSourceAdapterInterface
{
private ?PDO $connection = null;
private string $type = 'mysql';
/**
* 建立数据库连接
*
* @param array<string, mixed> $config 数据源配置
* @return bool 是否连接成功
*/
public function connect(array $config): bool
{
try {
$host = $config['host'] ?? '127.0.0.1';
$port = $config['port'] ?? 3306;
$database = $config['database'] ?? '';
$username = $config['username'] ?? '';
$password = $config['password'] ?? '';
$charset = $config['charset'] ?? 'utf8mb4';
// 构建 DSN
$dsn = "mysql:host={$host};port={$port};dbname={$database};charset={$charset}";
// PDO 选项
$options = [
PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
PDO::ATTR_EMULATE_PREPARES => false, // 禁用预处理语句模拟
PDO::ATTR_PERSISTENT => $config['persistent'] ?? false, // 是否持久连接
PDO::ATTR_TIMEOUT => $config['timeout'] ?? 10, // 连接超时
];
$this->connection = new PDO($dsn, $username, $password, $options);
LoggerHelper::logBusiness('mysql_adapter_connected', [
'host' => $host,
'port' => $port,
'database' => $database,
]);
return true;
} catch (PDOException $e) {
LoggerHelper::logError($e, [
'component' => 'MySQLAdapter',
'action' => 'connect',
'config' => array_merge($config, ['password' => '***']), // 隐藏密码
]);
return false;
}
}
/**
* 关闭数据库连接
*
* @return void
*/
public function disconnect(): void
{
if ($this->connection !== null) {
$this->connection = null;
LoggerHelper::logBusiness('mysql_adapter_disconnected', []);
}
}
/**
* 测试连接是否有效
*
* @return bool 连接是否有效
*/
public function isConnected(): bool
{
if ($this->connection === null) {
return false;
}
try {
// 执行简单查询测试连接
$this->connection->query('SELECT 1');
return true;
} catch (PDOException $e) {
LoggerHelper::logError($e, [
'component' => 'MySQLAdapter',
'action' => 'isConnected',
]);
return false;
}
}
/**
* 执行查询（返回多条记录）
*
* @param string $sql SQL 查询语句
* @param array<string, mixed> $params 查询参数（绑定参数）
* @return array<array<string, mixed>> 查询结果数组
*/
public function query(string $sql, array $params = []): array
{
if ($this->connection === null) {
throw new \RuntimeException('数据库连接未建立');
}
try {
$stmt = $this->connection->prepare($sql);
$stmt->execute($params);
$results = $stmt->fetchAll(PDO::FETCH_ASSOC);
LoggerHelper::logBusiness('mysql_query_executed', [
'sql' => $sql,
'params_count' => count($params),
'result_count' => count($results),
]);
return $results;
} catch (PDOException $e) {
LoggerHelper::logError($e, [
'component' => 'MySQLAdapter',
'action' => 'query',
'sql' => $sql,
'params' => $params,
]);
throw $e;
}
}
/**
* 执行查询（返回单条记录）
*
* @param string $sql SQL 查询语句
* @param array<string, mixed> $params 查询参数
* @return array<string, mixed>|null 查询结果（单条记录）或 null
*/
public function queryOne(string $sql, array $params = []): ?array
{
if ($this->connection === null) {
throw new \RuntimeException('数据库连接未建立');
}
try {
$stmt = $this->connection->prepare($sql);
$stmt->execute($params);
$result = $stmt->fetch(PDO::FETCH_ASSOC);
LoggerHelper::logBusiness('mysql_query_one_executed', [
'sql' => $sql,
'params_count' => count($params),
'has_result' => $result !== false,
]);
return $result !== false ? $result : null;
} catch (PDOException $e) {
LoggerHelper::logError($e, [
'component' => 'MySQLAdapter',
'action' => 'queryOne',
'sql' => $sql,
'params' => $params,
]);
throw $e;
}
}
/**
* 批量查询（分页查询，用于大数据量场景）
*
* @param string $sql SQL 查询语句（需要包含 LIMIT 和 OFFSET，或由适配器自动添加）
* @param array<string, mixed> $params 查询参数
* @param int $offset 偏移量
* @param int $limit 每页数量
* @return array<array<string, mixed>> 查询结果数组
*/
public function queryBatch(string $sql, array $params = [], int $offset = 0, int $limit = 1000): array
{
if ($this->connection === null) {
throw new \RuntimeException('数据库连接未建立');
}
try {
// 如果 SQL 中已包含 LIMIT，则直接使用；否则自动添加
if (stripos($sql, 'LIMIT') === false) {
$sql .= " LIMIT {$limit} OFFSET {$offset}";
}
$stmt = $this->connection->prepare($sql);
$stmt->execute($params);
$results = $stmt->fetchAll(PDO::FETCH_ASSOC);
LoggerHelper::logBusiness('mysql_query_batch_executed', [
'sql' => $sql,
'offset' => $offset,
'limit' => $limit,
'result_count' => count($results),
]);
return $results;
} catch (PDOException $e) {
LoggerHelper::logError($e, [
'component' => 'MySQLAdapter',
'action' => 'queryBatch',
'sql' => $sql,
'params' => $params,
'offset' => $offset,
'limit' => $limit,
]);
throw $e;
}
}
/**
* 获取数据源类型
*
* @return string 数据源类型
*/
public function getType(): string
{
return $this->type;
}
}

116
Moncter/app/service/DataSource/DataSourceAdapterFactory.php Normal file
View File

@@ -0,0 +1,116 @@
<?php
namespace app\service\DataSource;
use app\service\DataSource\Adapter\MySQLAdapter;
use app\service\DataSource\Adapter\MongoDBAdapter;
use app\utils\LoggerHelper;
/**
* 数据源适配器工厂
*
* 职责：
* - 根据数据源类型创建对应的适配器实例
* - 管理适配器实例（单例模式，避免重复创建连接）
*/
class DataSourceAdapterFactory
{
/**
* 适配器实例缓存（单例模式）
*
* @var array<string, DataSourceAdapterInterface>
*/
private static array $instances = [];
/**
* 创建数据源适配器
*
* @param string $type 数据源类型（mysql、postgresql、mongodb 等）
* @param array<string, mixed> $config 数据源配置
* @return DataSourceAdapterInterface 适配器实例
* @throws \InvalidArgumentException 不支持的数据源类型
*/
public static function create(string $type, array $config): DataSourceAdapterInterface
{
// 生成缓存键（基于类型和配置）
$cacheKey = self::generateCacheKey($type, $config);
// 如果已存在实例，直接返回
if (isset(self::$instances[$cacheKey])) {
$adapter = self::$instances[$cacheKey];
// 检查连接是否有效
if ($adapter->isConnected()) {
return $adapter;
}
// 连接已断开，重新创建
unset(self::$instances[$cacheKey]);
}
// 根据类型创建适配器
$adapter = match (strtolower($type)) {
'mysql' => new MySQLAdapter(),
'mongodb' => new MongoDBAdapter(),
// 'postgresql' => new PostgreSQLAdapter(),
default => throw new \InvalidArgumentException("不支持的数据源类型: {$type}"),
};
// 建立连接
if (!$adapter->connect($config)) {
throw new \RuntimeException("无法连接到数据源: {$type}");
}
// 缓存实例
self::$instances[$cacheKey] = $adapter;
LoggerHelper::logBusiness('data_source_adapter_created', [
'type' => $type,
'cache_key' => $cacheKey,
]);
return $adapter;
}
/**
* 生成缓存键
*
* @param string $type 数据源类型
* @param array<string, mixed> $config 数据源配置
* @return string 缓存键
*/
private static function generateCacheKey(string $type, array $config): string
{
// 基于类型、主机、端口、数据库名生成唯一键
$host = $config['host'] ?? 'unknown';
$port = $config['port'] ?? 'unknown';
$database = $config['database'] ?? 'unknown';
return md5("{$type}:{$host}:{$port}:{$database}");
}
/**
* 清除所有适配器实例（用于测试或重新连接）
*
* @return void
*/
public static function clearInstances(): void
{
foreach (self::$instances as $adapter) {
try {
$adapter->disconnect();
} catch (\Throwable $e) {
LoggerHelper::logError($e, ['component' => 'DataSourceAdapterFactory', 'action' => 'clearInstances']);
}
}
self::$instances = [];
}
/**
* 获取所有已创建的适配器实例
*
* @return array<string, DataSourceAdapterInterface>
*/
public static function getInstances(): array
{
return self::$instances;
}
}

73
Moncter/app/service/DataSource/DataSourceAdapterInterface.php Normal file
View File

@@ -0,0 +1,73 @@
<?php
namespace app\service\DataSource;
/**
* 数据源适配器接口
*
* 职责：
* - 定义统一的数据源访问接口
* - 支持多种数据库类型（MySQL、PostgreSQL、MongoDB 等）
* - 提供基础查询能力
*/
interface DataSourceAdapterInterface
{
/**
* 建立数据库连接
*
* @param array<string, mixed> $config 数据源配置
* @return bool 是否连接成功
*/
public function connect(array $config): bool;
/**
* 关闭数据库连接
*
* @return void
*/
public function disconnect(): void;
/**
* 测试连接是否有效
*
* @return bool 连接是否有效
*/
public function isConnected(): bool;
/**
* 执行查询（返回多条记录）
*
* @param string $sql SQL 查询语句（或 MongoDB 查询条件）
* @param array<string, mixed> $params 查询参数（绑定参数或 MongoDB 查询选项）
* @return array<array<string, mixed>> 查询结果数组
*/
public function query(string $sql, array $params = []): array;
/**
* 执行查询（返回单条记录）
*
* @param string $sql SQL 查询语句（或 MongoDB 查询条件）
* @param array<string, mixed> $params 查询参数
* @return array<string, mixed>|null 查询结果（单条记录）或 null
*/
public function queryOne(string $sql, array $params = []): ?array;
/**
* 批量查询（分页查询，用于大数据量场景）
*
* @param string $sql SQL 查询语句
* @param array<string, mixed> $params 查询参数
* @param int $offset 偏移量
* @param int $limit 每页数量
* @return array<array<string, mixed>> 查询结果数组
*/
public function queryBatch(string $sql, array $params = [], int $offset = 0, int $limit = 1000): array;
/**
* 获取数据源类型
*
* @return string 数据源类型（mysql、postgresql、mongodb 等）
*/
public function getType(): string;
}

68
Moncter/app/service/DataSource/PollingStrategyFactory.php Normal file
View File

@@ -0,0 +1,68 @@
<?php
namespace app\service\DataSource;
use app\service\DataSource\Strategy\DefaultConsumptionStrategy;
use app\utils\LoggerHelper;
/**
* 轮询策略工厂
*
* 职责：
* - 根据配置创建对应的轮询策略实例
* - 支持自定义策略类
*/
class PollingStrategyFactory
{
/**
* 创建轮询策略
*
* @param string|array<string, mixed> $strategyConfig 策略配置（字符串为策略类名，数组包含 class 和 config）
* @return PollingStrategyInterface 策略实例
* @throws \InvalidArgumentException 无效的策略配置
*/
public static function create(string|array $strategyConfig): PollingStrategyInterface
{
// 如果配置是字符串，则作为策略类名
if (is_string($strategyConfig)) {
$className = $strategyConfig;
$strategyConfig = ['class' => $className];
}
// 获取策略类名
$className = $strategyConfig['class'] ?? null;
if (!$className) {
// 如果没有指定策略，使用默认策略
$className = DefaultConsumptionStrategy::class;
}
// 验证类是否存在
if (!class_exists($className)) {
throw new \InvalidArgumentException("策略类不存在: {$className}");
}
// 验证类是否实现了接口
if (!is_subclass_of($className, PollingStrategyInterface::class)) {
throw new \InvalidArgumentException("策略类必须实现 PollingStrategyInterface: {$className}");
}
// 创建策略实例
try {
$strategy = new $className();
LoggerHelper::logBusiness('polling_strategy_created', [
'class' => $className,
]);
return $strategy;
} catch (\Throwable $e) {
LoggerHelper::logError($e, [
'component' => 'PollingStrategyFactory',
'action' => 'create',
'class' => $className,
]);
throw new \RuntimeException("无法创建策略实例: {$className}", 0, $e);
}
}
}

54
Moncter/app/service/DataSource/PollingStrategyInterface.php Normal file
View File

@@ -0,0 +1,54 @@
<?php
namespace app\service\DataSource;
/**
* 轮询策略接口
*
* 职责：
* - 定义自定义轮询业务逻辑的接口
* - 每个数据源可配置独立的轮询策略
* - 支持自定义查询、转换、验证逻辑
*/
interface PollingStrategyInterface
{
/**
* 执行轮询查询
*
* @param DataSourceAdapterInterface $adapter 数据源适配器
* @param array<string, mixed> $config 数据源配置
* @param array<string, mixed> $lastSyncInfo 上次同步信息（包含 last_sync_time、last_sync_id 等）
* @return array<array<string, mixed>> 查询结果数组（原始数据）
*/
public function poll(
DataSourceAdapterInterface $adapter,
array $config,
array $lastSyncInfo = []
): array;
/**
* 数据转换
*
* @param array<array<string, mixed>> $rawData 原始数据
* @param array<string, mixed> $config 数据源配置
* @return array<array<string, mixed>> 转换后的数据（标准格式）
*/
public function transform(array $rawData, array $config): array;
/**
* 数据验证
*
* @param array<string, mixed> $record 单条记录
* @param array<string, mixed> $config 数据源配置
* @return bool 是否通过验证
*/
public function validate(array $record, array $config): bool;
/**
* 获取策略名称
*
* @return string 策略名称
*/
public function getName(): string;
}

197
Moncter/app/service/DataSource/Strategy/DefaultConsumptionStrategy.php Normal file
View File

@@ -0,0 +1,197 @@
<?php
namespace app\service\DataSource\Strategy;
use app\service\DataSource\DataSourceAdapterInterface;
use app\service\DataSource\PollingStrategyInterface;
use app\utils\LoggerHelper;
/**
* 默认消费记录轮询策略（示例）
*
* 职责：
* - 提供默认的轮询策略实现示例
* - 展示如何实现自定义业务逻辑
* - 可根据实际需求扩展或替换
*/
class DefaultConsumptionStrategy implements PollingStrategyInterface
{
/**
* 执行轮询查询
*
* @param DataSourceAdapterInterface $adapter 数据源适配器
* @param array<string, mixed> $config 数据源配置
* @param array<string, mixed> $lastSyncInfo 上次同步信息
* @return array<array<string, mixed>> 查询结果数组
*/
public function poll(
DataSourceAdapterInterface $adapter,
array $config,
array $lastSyncInfo = []
): array {
// 从配置中获取表名和查询条件
$tableName = $config['table'] ?? 'consumption_records';
$lastSyncTime = $lastSyncInfo['last_sync_time'] ?? null;
$lastSyncId = $lastSyncInfo['last_sync_id'] ?? null;
// 构建 SQL 查询（增量查询）
$sql = "SELECT * FROM `{$tableName}` WHERE 1=1";
$params = [];
// 如果有上次同步时间，只查询新增或更新的记录
if ($lastSyncTime !== null) {
$sql .= " AND (`created_at` > :last_sync_time OR `updated_at` > :last_sync_time)";
$params[':last_sync_time'] = $lastSyncTime;
}
// 如果有上次同步ID，用于去重（可选）
if ($lastSyncId !== null) {
$sql .= " AND `id` > :last_sync_id";
$params[':last_sync_id'] = $lastSyncId;
}
// 按创建时间排序
$sql .= " ORDER BY `created_at` ASC, `id` ASC";
// 执行查询（批量查询，每次最多1000条）
$limit = $config['batch_size'] ?? 1000;
$offset = 0;
$allResults = [];
do {
$batchSql = $sql . " LIMIT {$limit} OFFSET {$offset}";
$results = $adapter->queryBatch($batchSql, $params, $offset, $limit);
if (empty($results)) {
break;
}
$allResults = array_merge($allResults, $results);
$offset += $limit;
// 防止无限循环（最多查询10万条）
if (count($allResults) >= 100000) {
LoggerHelper::logBusiness('polling_batch_limit_reached', [
'table' => $tableName,
'count' => count($allResults),
]);
break;
}
} while (count($results) === $limit);
LoggerHelper::logBusiness('polling_query_completed', [
'table' => $tableName,
'result_count' => count($allResults),
'last_sync_time' => $lastSyncTime,
]);
return $allResults;
}
/**
* 数据转换
*
* @param array<array<string, mixed>> $rawData 原始数据
* @param array<string, mixed> $config 数据源配置
* @return array<array<string, mixed>> 转换后的数据
*/
public function transform(array $rawData, array $config): array
{
// 字段映射配置（从外部数据库字段映射到标准字段）
$fieldMapping = $config['field_mapping'] ?? [
// 默认映射（如果外部数据库字段名与标准字段名一致，则无需映射）
'id' => 'id',
'user_id' => 'user_id',
'amount' => 'amount',
'store_id' => 'store_id',
'product_id' => 'product_id',
'consume_time' => 'consume_time',
'created_at' => 'created_at',
];
$transformedData = [];
foreach ($rawData as $record) {
$transformed = [];
// 应用字段映射
foreach ($fieldMapping as $standardField => $sourceField) {
if (isset($record[$sourceField])) {
$transformed[$standardField] = $record[$sourceField];
}
}
// 确保必要字段存在
if (!empty($transformed)) {
$transformedData[] = $transformed;
}
}
LoggerHelper::logBusiness('polling_transform_completed', [
'input_count' => count($rawData),
'output_count' => count($transformedData),
]);
return $transformedData;
}
/**
* 数据验证
*
* @param array<string, mixed> $record 单条记录
* @param array<string, mixed> $config 数据源配置
* @return bool 是否通过验证
*/
public function validate(array $record, array $config): bool
{
// 必填字段验证
$requiredFields = $config['required_fields'] ?? ['user_id', 'amount', 'consume_time'];
foreach ($requiredFields as $field) {
if (!isset($record[$field]) || $record[$field] === null || $record[$field] === '') {
LoggerHelper::logBusiness('polling_validation_failed', [
'reason' => "缺少必填字段: {$field}",
'record' => $record,
]);
return false;
}
}
// 金额验证（必须为正数）
if (isset($record['amount'])) {
$amount = (float)$record['amount'];
if ($amount <= 0) {
LoggerHelper::logBusiness('polling_validation_failed', [
'reason' => '金额必须大于0',
'amount' => $amount,
]);
return false;
}
}
// 时间格式验证（可选）
if (isset($record['consume_time'])) {
$time = strtotime($record['consume_time']);
if ($time === false) {
LoggerHelper::logBusiness('polling_validation_failed', [
'reason' => '时间格式无效',
'consume_time' => $record['consume_time'],
]);
return false;
}
}
return true;
}
/**
* 获取策略名称
*
* @return string 策略名称
*/
public function getName(): string
{
return 'default_consumption';
}
}

225
Moncter/app/service/DataSource/Strategy/MongoDBConsumptionStrategy.php Normal file
View File

@@ -0,0 +1,225 @@
<?php
namespace app\service\DataSource\Strategy;
use app\service\DataSource\DataSourceAdapterInterface;
use app\service\DataSource\PollingStrategyInterface;
use app\utils\LoggerHelper;
/**
* MongoDB 消费记录轮询策略
*
* 职责：
* - 提供 MongoDB 专用的轮询策略实现
* - 展示如何实现自定义业务逻辑
*/
class MongoDBConsumptionStrategy implements PollingStrategyInterface
{
/**
* 执行轮询查询
*
* @param DataSourceAdapterInterface $adapter 数据源适配器
* @param array<string, mixed> $config 数据源配置
* @param array<string, mixed> $lastSyncInfo 上次同步信息
* @return array<array<string, mixed>> 查询结果数组
*/
public function poll(
DataSourceAdapterInterface $adapter,
array $config,
array $lastSyncInfo = []
): array {
// 从配置中获取集合名和查询条件
$collectionName = $config['collection'] ?? 'consumption_records';
$lastSyncTime = $lastSyncInfo['last_sync_time'] ?? null;
$lastSyncId = $lastSyncInfo['last_sync_id'] ?? null;
// 构建 MongoDB 查询过滤器
$filter = [];
// 如果有上次同步时间，只查询新增或更新的记录
if ($lastSyncTime !== null) {
$lastSyncTimestamp = is_numeric($lastSyncTime) ? (int)$lastSyncTime : strtotime($lastSyncTime);
$lastSyncDate = new \MongoDB\BSON\UTCDateTime($lastSyncTimestamp * 1000);
$filter['$or'] = [
['created_at' => ['$gt' => $lastSyncDate]],
['updated_at' => ['$gt' => $lastSyncDate]],
];
}
// 如果有上次同步ID，用于去重（可选）
if ($lastSyncId !== null) {
$filter['_id'] = ['$gt' => $lastSyncId];
}
// 查询选项
$options = [
'sort' => ['created_at' => 1, '_id' => 1], // 按创建时间和ID排序
];
// 执行查询（批量查询，每次最多1000条）
$limit = $config['batch_size'] ?? 1000;
$offset = 0;
$allResults = [];
do {
// MongoDB 适配器的 queryBatch 方法签名：queryBatch(string $sql, array $params = [], int $offset = 0, int $limit = 1000)
// 对于 MongoDB，$sql 是集合名，$params 包含 'filter' 和 'options'
$results = $adapter->queryBatch($collectionName, [
'filter' => $filter,
'options' => $options,
], $offset, $limit);
if (empty($results)) {
break;
}
$allResults = array_merge($allResults, $results);
$offset += $limit;
// 防止无限循环（最多查询10万条）
if (count($allResults) >= 100000) {
LoggerHelper::logBusiness('polling_batch_limit_reached', [
'collection' => $collectionName,
'count' => count($allResults),
]);
break;
}
} while (count($results) === $limit);
LoggerHelper::logBusiness('polling_query_completed', [
'collection' => $collectionName,
'result_count' => count($allResults),
'last_sync_time' => $lastSyncTime,
]);
return $allResults;
}
/**
* 数据转换
*
* @param array<array<string, mixed>> $rawData 原始数据
* @param array<string, mixed> $config 数据源配置
* @return array<array<string, mixed>> 转换后的数据
*/
public function transform(array $rawData, array $config): array
{
// 字段映射配置（从外部数据库字段映射到标准字段）
$fieldMapping = $config['field_mapping'] ?? [
// 默认映射（MongoDB 使用 _id，需要转换为 id）
'_id' => 'id',
'user_id' => 'user_id',
'amount' => 'amount',
'store_id' => 'store_id',
'product_id' => 'product_id',
'consume_time' => 'consume_time',
'created_at' => 'created_at',
];
$transformedData = [];
foreach ($rawData as $record) {
$transformed = [];
// 处理 MongoDB 的 _id 字段（转换为字符串）
if (isset($record['_id'])) {
if (is_object($record['_id']) && method_exists($record['_id'], '__toString')) {
$record['id'] = (string)$record['_id'];
} else {
$record['id'] = (string)$record['_id'];
}
}
// 处理 MongoDB 的日期字段（UTCDateTime 转换为字符串）
foreach (['created_at', 'updated_at', 'consume_time'] as $dateField) {
if (isset($record[$dateField])) {
if (is_object($record[$dateField]) && method_exists($record[$dateField], 'toDateTime')) {
$record[$dateField] = $record[$dateField]->toDateTime()->format('Y-m-d H:i:s');
} elseif (is_object($record[$dateField]) && method_exists($record[$dateField], '__toString')) {
$record[$dateField] = (string)$record[$dateField];
}
}
}
// 应用字段映射
foreach ($fieldMapping as $standardField => $sourceField) {
if (isset($record[$sourceField])) {
$transformed[$standardField] = $record[$sourceField];
}
}
// 确保必要字段存在
if (!empty($transformed)) {
$transformedData[] = $transformed;
}
}
LoggerHelper::logBusiness('polling_transform_completed', [
'input_count' => count($rawData),
'output_count' => count($transformedData),
]);
return $transformedData;
}
/**
* 数据验证
*
* @param array<string, mixed> $record 单条记录
* @param array<string, mixed> $config 数据源配置
* @return bool 是否通过验证
*/
public function validate(array $record, array $config): bool
{
// 必填字段验证
$requiredFields = $config['required_fields'] ?? ['user_id', 'amount', 'consume_time'];
foreach ($requiredFields as $field) {
if (!isset($record[$field]) || $record[$field] === null || $record[$field] === '') {
LoggerHelper::logBusiness('polling_validation_failed', [
'reason' => "缺少必填字段: {$field}",
'record' => $record,
]);
return false;
}
}
// 金额验证（必须为正数）
if (isset($record['amount'])) {
$amount = (float)$record['amount'];
if ($amount <= 0) {
LoggerHelper::logBusiness('polling_validation_failed', [
'reason' => '金额必须大于0',
'amount' => $amount,
]);
return false;
}
}
// 时间格式验证（可选）
if (isset($record['consume_time'])) {
$time = strtotime($record['consume_time']);
if ($time === false) {
LoggerHelper::logBusiness('polling_validation_failed', [
'reason' => '时间格式无效',
'consume_time' => $record['consume_time'],
]);
return false;
}
}
return true;
}
/**
* 获取策略名称
*
* @return string 策略名称
*/
public function getName(): string
{
return 'mongodb_consumption';
}
}