From 93ccc23e2afcbbb9ca8792e1ebb5901f9b94e4d7 Mon Sep 17 00:00:00 2001 From: Sosokker Date: Mon, 12 May 2025 14:26:27 +0700 Subject: [PATCH] ruff format and pyright lint --- pipeline/models.py | 69 ++++++++++++++++++++-------------------------- 1 file changed, 30 insertions(+), 39 deletions(-) diff --git a/pipeline/models.py b/pipeline/models.py index b40fb61..084d953 100644 --- a/pipeline/models.py +++ b/pipeline/models.py @@ -13,6 +13,7 @@ class RunCreate(BaseModel): """ Model for creating a new run. (Empty) """ + pass @@ -20,9 +21,10 @@ class Run(BaseModel): """ Status of a pipeline run. """ + id: UUID pipeline_id: UUID - status: Literal['PENDING', 'RUNNING', 'COMPLETED', 'FAILED'] + status: Literal["PENDING", "RUNNING", "COMPLETED", "FAILED"] started_at: datetime finished_at: Optional[datetime] = None @@ -31,24 +33,20 @@ class RunResult(Run): """ Extended run model including results or error. """ + results: Optional[List[Dict[str, Any]]] = None error: Optional[str] = None - class ApiConfig(BaseModel): """ Configuration for an API source. """ - url: HttpUrl = Field( - ..., - description="API endpoint URL", - example="https://api.example.com/data" - ) + + url: HttpUrl = Field(..., description="API endpoint URL") token: Optional[str] = Field( None, description="Optional bearer token for API authentication", - example="abcdef123456" ) @@ -56,20 +54,18 @@ class ScrapeConfig(BaseModel): """ Configuration for a web-scraping source. """ + urls: List[HttpUrl] = Field( ..., description="List of URLs to scrape", - example=["https://example.com/page1", "https://example.com/page2"] ) schema_file: Optional[str] = Field( None, description="Path to a JSON file containing CSS extraction schema", - example="schemas/page_schema.json" ) prompt: Optional[str] = Field( None, description="Prompt string for LLM-based extraction", - example="Extract product titles and prices" ) @@ -77,25 +73,21 @@ class FileConfig(BaseModel): """ Configuration for a file-based source. Supports either a file path or an uploaded file. """ + path: Optional[str] = Field( None, description="Path to the input file (optional if upload is provided)", - example="/data/myfile.json" ) upload: Optional[Any] = Field( None, description="Uploaded file object or metadata (optional if path is provided)", - example=None ) upload_filename: Optional[str] = Field( None, description="Original filename of the uploaded file (for validation)", - example="myfile.json" ) format: Literal["csv", "json", "sqlite"] = Field( - "json", - description="Format of the file", - example="csv" + "json", description="Format of the file" ) @field_validator("path", mode="before") @@ -123,13 +115,14 @@ class FileConfig(BaseModel): return v - class ApiSource(BaseModel): """ An API-based data source. """ + type: Literal["api"] = Field( - "api", description="Discriminator for API source" # Removed const=True + "api", + description="Discriminator for API source", # Removed const=True ) config: ApiConfig @@ -138,8 +131,10 @@ class ScrapeSource(BaseModel): """ A web-scraping data source. """ + type: Literal["scrape"] = Field( - "scrape", description="Discriminator for scrape source" # Removed const=True + "scrape", + description="Discriminator for scrape source", # Removed const=True ) config: ScrapeConfig @@ -148,28 +143,31 @@ class FileSource(BaseModel): """ A file-based data source. """ + type: Literal["file"] = Field( - "file", description="Discriminator for file source" # Removed const=True + "file", + description="Discriminator for file source", # Removed const=True ) config: FileConfig + Source = Annotated[ Union[ApiSource, ScrapeSource, FileSource], - Field(discriminator="type", description="Union of all source types") + Field(discriminator="type", description="Union of all source types"), ] + class PipelineCreate(BaseModel): """ Payload for creating a new pipeline. """ + name: Optional[str] = Field( - None, + default=None, description="Optional human-readable name for the pipeline", - example="My Data Pipeline" ) sources: List[Source] = Field( - ..., - description="List of data sources for this pipeline" + ..., description="List of data sources for this pipeline" ) @@ -177,19 +175,12 @@ class Pipeline(BaseModel): """ Representation of a pipeline. """ - id: UUID = Field( - ..., - description="Unique identifier for the pipeline" - ) + + id: UUID = Field(..., description="Unique identifier for the pipeline") name: Optional[str] = Field( - None, - description="Optional human-readable name for the pipeline" - ) - sources: List[Source] = Field( - ..., - description="List of configured data sources" + None, description="Optional human-readable name for the pipeline" ) + sources: List[Source] = Field(..., description="List of configured data sources") created_at: datetime = Field( - ..., - description="UTC timestamp when the pipeline was created" - ) \ No newline at end of file + ..., description="UTC timestamp when the pipeline was created" + )