fix: 统一存储
This commit is contained in:
root
17
python_core/storage/__init__.py
Normal file
17
python_core/storage/__init__.py
Normal file
@@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
存储层模块
|
||||
提供统一的存储接口，支持多种存储后端
|
||||
"""
|
||||
|
||||
from .base import StorageInterface, StorageConfig
|
||||
from .json_storage import JSONStorage
|
||||
from .factory import StorageFactory, get_storage
|
||||
|
||||
__all__ = [
|
||||
"StorageInterface",
|
||||
"StorageConfig",
|
||||
"JSONStorage",
|
||||
"StorageFactory",
|
||||
"get_storage"
|
||||
]
|
||||
199
python_core/storage/base.py
Normal file
199
python_core/storage/base.py
Normal file
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
存储接口基类
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, List, Dict, Optional
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
|
||||
class StorageType(Enum):
|
||||
"""存储类型"""
|
||||
JSON = "json"
|
||||
DATABASE = "database"
|
||||
MONGODB = "mongodb"
|
||||
REDIS = "redis"
|
||||
|
||||
@dataclass
|
||||
class StorageConfig:
|
||||
"""存储配置"""
|
||||
storage_type: StorageType
|
||||
connection_string: Optional[str] = None
|
||||
base_path: Optional[str] = None
|
||||
database_name: Optional[str] = None
|
||||
table_prefix: Optional[str] = None
|
||||
|
||||
# JSON存储配置
|
||||
json_indent: int = 2
|
||||
json_ensure_ascii: bool = False
|
||||
|
||||
# 数据库配置
|
||||
pool_size: int = 10
|
||||
max_overflow: int = 20
|
||||
|
||||
# MongoDB配置
|
||||
collection_prefix: str = ""
|
||||
|
||||
# Redis配置
|
||||
redis_db: int = 0
|
||||
redis_expire: Optional[int] = None
|
||||
|
||||
class StorageInterface(ABC):
|
||||
"""存储接口基类"""
|
||||
|
||||
def __init__(self, config: StorageConfig):
|
||||
self.config = config
|
||||
|
||||
@abstractmethod
|
||||
def save(self, collection: str, key: str, data: Any) -> bool:
|
||||
"""保存数据
|
||||
|
||||
Args:
|
||||
collection: 集合/表名
|
||||
key: 数据键
|
||||
data: 要保存的数据
|
||||
|
||||
Returns:
|
||||
bool: 保存是否成功
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def load(self, collection: str, key: str) -> Any:
|
||||
"""加载数据
|
||||
|
||||
Args:
|
||||
collection: 集合/表名
|
||||
key: 数据键
|
||||
|
||||
Returns:
|
||||
Any: 加载的数据，不存在时返回None
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def delete(self, collection: str, key: str) -> bool:
|
||||
"""删除数据
|
||||
|
||||
Args:
|
||||
collection: 集合/表名
|
||||
key: 数据键
|
||||
|
||||
Returns:
|
||||
bool: 删除是否成功
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def exists(self, collection: str, key: str) -> bool:
|
||||
"""检查数据是否存在
|
||||
|
||||
Args:
|
||||
collection: 集合/表名
|
||||
key: 数据键
|
||||
|
||||
Returns:
|
||||
bool: 数据是否存在
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def list_keys(self, collection: str, pattern: str = "*") -> List[str]:
|
||||
"""列出所有键
|
||||
|
||||
Args:
|
||||
collection: 集合/表名
|
||||
pattern: 键的模式匹配
|
||||
|
||||
Returns:
|
||||
List[str]: 键列表
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def save_batch(self, collection: str, data: Dict[str, Any]) -> bool:
|
||||
"""批量保存数据
|
||||
|
||||
Args:
|
||||
collection: 集合/表名
|
||||
data: 键值对数据
|
||||
|
||||
Returns:
|
||||
bool: 保存是否成功
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def load_batch(self, collection: str, keys: List[str]) -> Dict[str, Any]:
|
||||
"""批量加载数据
|
||||
|
||||
Args:
|
||||
collection: 集合/表名
|
||||
keys: 键列表
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 键值对数据
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def clear_collection(self, collection: str) -> bool:
|
||||
"""清空集合
|
||||
|
||||
Args:
|
||||
collection: 集合/表名
|
||||
|
||||
Returns:
|
||||
bool: 清空是否成功
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_collections(self) -> List[str]:
|
||||
"""获取所有集合名称
|
||||
|
||||
Returns:
|
||||
List[str]: 集合名称列表
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_stats(self, collection: str) -> Dict[str, Any]:
|
||||
"""获取集合统计信息
|
||||
|
||||
Args:
|
||||
collection: 集合/表名
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 统计信息
|
||||
"""
|
||||
pass
|
||||
|
||||
def close(self):
|
||||
"""关闭存储连接"""
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
"""上下文管理器入口"""
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
"""上下文管理器出口"""
|
||||
self.close()
|
||||
|
||||
class StorageException(Exception):
|
||||
"""存储异常基类"""
|
||||
pass
|
||||
|
||||
class StorageConnectionError(StorageException):
|
||||
"""存储连接错误"""
|
||||
pass
|
||||
|
||||
class StorageOperationError(StorageException):
|
||||
"""存储操作错误"""
|
||||
pass
|
||||
|
||||
class StorageNotFoundError(StorageException):
|
||||
"""存储数据未找到错误"""
|
||||
pass
|
||||
155
python_core/storage/factory.py
Normal file
155
python_core/storage/factory.py
Normal file
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
存储工厂类
|
||||
"""
|
||||
|
||||
from typing import Type, Dict
|
||||
from .base import StorageInterface, StorageConfig, StorageType, StorageException
|
||||
from .json_storage import JSONStorage
|
||||
from python_core.utils.logger import logger
|
||||
|
||||
class StorageFactory:
|
||||
"""存储工厂类"""
|
||||
|
||||
_storage_classes: Dict[StorageType, Type[StorageInterface]] = {
|
||||
StorageType.JSON: JSONStorage,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def register_storage(cls, storage_type: StorageType, storage_class: Type[StorageInterface]):
|
||||
"""注册存储实现
|
||||
|
||||
Args:
|
||||
storage_type: 存储类型
|
||||
storage_class: 存储实现类
|
||||
"""
|
||||
cls._storage_classes[storage_type] = storage_class
|
||||
logger.info(f"Registered storage type: {storage_type.value}")
|
||||
|
||||
@classmethod
|
||||
def create_storage(cls, config: StorageConfig) -> StorageInterface:
|
||||
"""创建存储实例
|
||||
|
||||
Args:
|
||||
config: 存储配置
|
||||
|
||||
Returns:
|
||||
StorageInterface: 存储实例
|
||||
|
||||
Raises:
|
||||
StorageException: 当存储类型不支持时
|
||||
"""
|
||||
storage_type = config.storage_type
|
||||
|
||||
if storage_type not in cls._storage_classes:
|
||||
available_types = list(cls._storage_classes.keys())
|
||||
raise StorageException(
|
||||
f"Unsupported storage type: {storage_type.value}. "
|
||||
f"Available types: {[t.value for t in available_types]}"
|
||||
)
|
||||
|
||||
storage_class = cls._storage_classes[storage_type]
|
||||
|
||||
try:
|
||||
storage = storage_class(config)
|
||||
logger.info(f"Created storage instance: {storage_type.value}")
|
||||
return storage
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create storage instance {storage_type.value}: {e}")
|
||||
raise StorageException(f"Failed to create storage: {e}")
|
||||
|
||||
@classmethod
|
||||
def get_available_types(cls) -> list[StorageType]:
|
||||
"""获取可用的存储类型
|
||||
|
||||
Returns:
|
||||
List[StorageType]: 可用的存储类型列表
|
||||
"""
|
||||
return list(cls._storage_classes.keys())
|
||||
|
||||
@classmethod
|
||||
def create_json_storage(cls, base_path: str = None) -> StorageInterface:
|
||||
"""创建JSON存储实例（便捷方法）
|
||||
|
||||
Args:
|
||||
base_path: 存储基础路径
|
||||
|
||||
Returns:
|
||||
StorageInterface: JSON存储实例
|
||||
"""
|
||||
config = StorageConfig(
|
||||
storage_type=StorageType.JSON,
|
||||
base_path=base_path
|
||||
)
|
||||
return cls.create_storage(config)
|
||||
|
||||
# 全局存储实例缓存
|
||||
_storage_instances: Dict[str, StorageInterface] = {}
|
||||
|
||||
def get_storage(storage_key: str = "default", config: StorageConfig = None) -> StorageInterface:
|
||||
"""获取存储实例（单例模式）
|
||||
|
||||
Args:
|
||||
storage_key: 存储实例键
|
||||
config: 存储配置（首次创建时需要）
|
||||
|
||||
Returns:
|
||||
StorageInterface: 存储实例
|
||||
|
||||
Raises:
|
||||
StorageException: 当配置缺失时
|
||||
"""
|
||||
if storage_key not in _storage_instances:
|
||||
if config is None:
|
||||
# 使用默认JSON存储配置
|
||||
config = StorageConfig(storage_type=StorageType.JSON)
|
||||
|
||||
_storage_instances[storage_key] = StorageFactory.create_storage(config)
|
||||
|
||||
return _storage_instances[storage_key]
|
||||
|
||||
def close_all_storages():
|
||||
"""关闭所有存储实例"""
|
||||
for storage_key, storage in _storage_instances.items():
|
||||
try:
|
||||
storage.close()
|
||||
logger.info(f"Closed storage instance: {storage_key}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to close storage {storage_key}: {e}")
|
||||
|
||||
_storage_instances.clear()
|
||||
|
||||
# 注册未来的存储实现
|
||||
def register_database_storage():
|
||||
"""注册数据库存储（未来实现）"""
|
||||
try:
|
||||
from .database_storage import DatabaseStorage
|
||||
StorageFactory.register_storage(StorageType.DATABASE, DatabaseStorage)
|
||||
except ImportError:
|
||||
logger.debug("Database storage not available")
|
||||
|
||||
def register_mongodb_storage():
|
||||
"""注册MongoDB存储（未来实现）"""
|
||||
try:
|
||||
from .mongodb_storage import MongoDBStorage
|
||||
StorageFactory.register_storage(StorageType.MONGODB, MongoDBStorage)
|
||||
except ImportError:
|
||||
logger.debug("MongoDB storage not available")
|
||||
|
||||
def register_redis_storage():
|
||||
"""注册Redis存储（未来实现）"""
|
||||
try:
|
||||
from .redis_storage import RedisStorage
|
||||
StorageFactory.register_storage(StorageType.REDIS, RedisStorage)
|
||||
except ImportError:
|
||||
logger.debug("Redis storage not available")
|
||||
|
||||
# 自动注册可用的存储类型
|
||||
def auto_register_storages():
|
||||
"""自动注册所有可用的存储类型"""
|
||||
register_database_storage()
|
||||
register_mongodb_storage()
|
||||
register_redis_storage()
|
||||
|
||||
# 模块加载时自动注册
|
||||
auto_register_storages()
|
||||
263
python_core/storage/json_storage.py
Normal file
263
python_core/storage/json_storage.py
Normal file
@@ -0,0 +1,263 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
JSON文件存储实现
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import glob
|
||||
from pathlib import Path
|
||||
from typing import Any, List, Dict
|
||||
from datetime import datetime
|
||||
|
||||
from .base import StorageInterface, StorageConfig, StorageException, StorageOperationError
|
||||
from python_core.utils.logger import logger
|
||||
|
||||
class JSONStorage(StorageInterface):
|
||||
"""JSON文件存储实现"""
|
||||
|
||||
def __init__(self, config: StorageConfig):
|
||||
super().__init__(config)
|
||||
|
||||
# 设置基础路径
|
||||
if config.base_path:
|
||||
self.base_path = Path(config.base_path)
|
||||
else:
|
||||
from python_core.config import settings
|
||||
self.base_path = settings.temp_dir / "storage"
|
||||
|
||||
# 确保目录存在
|
||||
self.base_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.info(f"JSON storage initialized at: {self.base_path}")
|
||||
|
||||
def _get_collection_path(self, collection: str) -> Path:
|
||||
"""获取集合目录路径"""
|
||||
return self.base_path / collection
|
||||
|
||||
def _get_file_path(self, collection: str, key: str) -> Path:
|
||||
"""获取文件路径"""
|
||||
collection_path = self._get_collection_path(collection)
|
||||
collection_path.mkdir(parents=True, exist_ok=True)
|
||||
return collection_path / f"{key}.json"
|
||||
|
||||
def _ensure_serializable(self, data: Any) -> Any:
|
||||
"""确保数据可序列化"""
|
||||
if hasattr(data, '__dict__'):
|
||||
# 处理dataclass或自定义对象
|
||||
if hasattr(data, '__dataclass_fields__'):
|
||||
from dataclasses import asdict
|
||||
return asdict(data)
|
||||
else:
|
||||
return data.__dict__
|
||||
elif isinstance(data, (list, tuple)):
|
||||
return [self._ensure_serializable(item) for item in data]
|
||||
elif isinstance(data, dict):
|
||||
return {k: self._ensure_serializable(v) for k, v in data.items()}
|
||||
else:
|
||||
return data
|
||||
|
||||
def save(self, collection: str, key: str, data: Any) -> bool:
|
||||
"""保存数据到JSON文件"""
|
||||
try:
|
||||
file_path = self._get_file_path(collection, key)
|
||||
|
||||
# 准备保存的数据
|
||||
save_data = {
|
||||
"key": key,
|
||||
"data": self._ensure_serializable(data),
|
||||
"created_at": datetime.now().isoformat(),
|
||||
"updated_at": datetime.now().isoformat()
|
||||
}
|
||||
|
||||
# 如果文件已存在，保留创建时间
|
||||
if file_path.exists():
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
existing_data = json.load(f)
|
||||
save_data["created_at"] = existing_data.get("created_at", save_data["created_at"])
|
||||
except Exception:
|
||||
pass # 如果读取失败，使用新的创建时间
|
||||
|
||||
# 保存文件
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(save_data, f,
|
||||
indent=self.config.json_indent,
|
||||
ensure_ascii=self.config.json_ensure_ascii)
|
||||
|
||||
logger.debug(f"Saved data to {file_path}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to save data to {collection}/{key}: {e}")
|
||||
raise StorageOperationError(f"Failed to save data: {e}")
|
||||
|
||||
def load(self, collection: str, key: str) -> Any:
|
||||
"""从JSON文件加载数据"""
|
||||
try:
|
||||
file_path = self._get_file_path(collection, key)
|
||||
|
||||
if not file_path.exists():
|
||||
return None
|
||||
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
file_data = json.load(f)
|
||||
return file_data.get("data")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load data from {collection}/{key}: {e}")
|
||||
raise StorageOperationError(f"Failed to load data: {e}")
|
||||
|
||||
def delete(self, collection: str, key: str) -> bool:
|
||||
"""删除JSON文件"""
|
||||
try:
|
||||
file_path = self._get_file_path(collection, key)
|
||||
|
||||
if file_path.exists():
|
||||
file_path.unlink()
|
||||
logger.debug(f"Deleted file {file_path}")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete data from {collection}/{key}: {e}")
|
||||
raise StorageOperationError(f"Failed to delete data: {e}")
|
||||
|
||||
def exists(self, collection: str, key: str) -> bool:
|
||||
"""检查JSON文件是否存在"""
|
||||
file_path = self._get_file_path(collection, key)
|
||||
return file_path.exists()
|
||||
|
||||
def list_keys(self, collection: str, pattern: str = "*") -> List[str]:
|
||||
"""列出集合中的所有键"""
|
||||
try:
|
||||
collection_path = self._get_collection_path(collection)
|
||||
|
||||
if not collection_path.exists():
|
||||
return []
|
||||
|
||||
# 使用glob模式匹配
|
||||
if pattern == "*":
|
||||
pattern = "*.json"
|
||||
elif not pattern.endswith(".json"):
|
||||
pattern = f"{pattern}.json"
|
||||
|
||||
files = glob.glob(str(collection_path / pattern))
|
||||
keys = [Path(f).stem for f in files]
|
||||
|
||||
return sorted(keys)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to list keys in {collection}: {e}")
|
||||
raise StorageOperationError(f"Failed to list keys: {e}")
|
||||
|
||||
def save_batch(self, collection: str, data: Dict[str, Any]) -> bool:
|
||||
"""批量保存数据"""
|
||||
try:
|
||||
success_count = 0
|
||||
for key, value in data.items():
|
||||
if self.save(collection, key, value):
|
||||
success_count += 1
|
||||
|
||||
logger.info(f"Batch saved {success_count}/{len(data)} items to {collection}")
|
||||
return success_count == len(data)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to batch save data to {collection}: {e}")
|
||||
raise StorageOperationError(f"Failed to batch save data: {e}")
|
||||
|
||||
def load_batch(self, collection: str, keys: List[str]) -> Dict[str, Any]:
|
||||
"""批量加载数据"""
|
||||
try:
|
||||
result = {}
|
||||
for key in keys:
|
||||
data = self.load(collection, key)
|
||||
if data is not None:
|
||||
result[key] = data
|
||||
|
||||
logger.debug(f"Batch loaded {len(result)}/{len(keys)} items from {collection}")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to batch load data from {collection}: {e}")
|
||||
raise StorageOperationError(f"Failed to batch load data: {e}")
|
||||
|
||||
def clear_collection(self, collection: str) -> bool:
|
||||
"""清空集合（删除所有文件）"""
|
||||
try:
|
||||
collection_path = self._get_collection_path(collection)
|
||||
|
||||
if not collection_path.exists():
|
||||
return True
|
||||
|
||||
# 删除所有JSON文件
|
||||
json_files = list(collection_path.glob("*.json"))
|
||||
for file_path in json_files:
|
||||
file_path.unlink()
|
||||
|
||||
logger.info(f"Cleared {len(json_files)} files from collection {collection}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to clear collection {collection}: {e}")
|
||||
raise StorageOperationError(f"Failed to clear collection: {e}")
|
||||
|
||||
def get_collections(self) -> List[str]:
|
||||
"""获取所有集合名称"""
|
||||
try:
|
||||
if not self.base_path.exists():
|
||||
return []
|
||||
|
||||
collections = []
|
||||
for item in self.base_path.iterdir():
|
||||
if item.is_dir():
|
||||
collections.append(item.name)
|
||||
|
||||
return sorted(collections)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get collections: {e}")
|
||||
raise StorageOperationError(f"Failed to get collections: {e}")
|
||||
|
||||
def get_stats(self, collection: str) -> Dict[str, Any]:
|
||||
"""获取集合统计信息"""
|
||||
try:
|
||||
collection_path = self._get_collection_path(collection)
|
||||
|
||||
if not collection_path.exists():
|
||||
return {
|
||||
"collection": collection,
|
||||
"exists": False,
|
||||
"file_count": 0,
|
||||
"total_size": 0
|
||||
}
|
||||
|
||||
json_files = list(collection_path.glob("*.json"))
|
||||
total_size = sum(f.stat().st_size for f in json_files)
|
||||
|
||||
# 获取最新和最旧的文件时间
|
||||
if json_files:
|
||||
file_times = [f.stat().st_mtime for f in json_files]
|
||||
oldest_file = min(file_times)
|
||||
newest_file = max(file_times)
|
||||
else:
|
||||
oldest_file = newest_file = None
|
||||
|
||||
return {
|
||||
"collection": collection,
|
||||
"exists": True,
|
||||
"file_count": len(json_files),
|
||||
"total_size": total_size,
|
||||
"oldest_file": datetime.fromtimestamp(oldest_file).isoformat() if oldest_file else None,
|
||||
"newest_file": datetime.fromtimestamp(newest_file).isoformat() if newest_file else None,
|
||||
"path": str(collection_path)
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get stats for collection {collection}: {e}")
|
||||
raise StorageOperationError(f"Failed to get stats: {e}")
|
||||
|
||||
def close(self):
|
||||
"""关闭存储连接（JSON存储无需特殊关闭操作）"""
|
||||
logger.debug("JSON storage closed")
|
||||
Reference in New Issue
Block a user