diff --git a/__pycache__/config.cpython-312.pyc b/__pycache__/config.cpython-312.pyc new file mode 100644 index 0000000..cbd389e Binary files /dev/null and b/__pycache__/config.cpython-312.pyc differ diff --git a/__pycache__/crawler_logic.cpython-312.pyc b/__pycache__/crawler_logic.cpython-312.pyc new file mode 100644 index 0000000..084ffd0 Binary files /dev/null and b/__pycache__/crawler_logic.cpython-312.pyc differ diff --git a/__pycache__/log_stream.cpython-312.pyc b/__pycache__/log_stream.cpython-312.pyc new file mode 100644 index 0000000..1f41388 Binary files /dev/null and b/__pycache__/log_stream.cpython-312.pyc differ diff --git a/__pycache__/logging_utils.cpython-312.pyc b/__pycache__/logging_utils.cpython-312.pyc new file mode 100644 index 0000000..1edfd97 Binary files /dev/null and b/__pycache__/logging_utils.cpython-312.pyc differ diff --git a/__pycache__/main.cpython-312.pyc b/__pycache__/main.cpython-312.pyc new file mode 100644 index 0000000..1b9524a Binary files /dev/null and b/__pycache__/main.cpython-312.pyc differ diff --git a/__pycache__/models.cpython-312.pyc b/__pycache__/models.cpython-312.pyc new file mode 100644 index 0000000..a3adcff Binary files /dev/null and b/__pycache__/models.cpython-312.pyc differ diff --git a/__pycache__/services.cpython-312.pyc b/__pycache__/services.cpython-312.pyc new file mode 100644 index 0000000..09c532e Binary files /dev/null and b/__pycache__/services.cpython-312.pyc differ diff --git a/__pycache__/stores.cpython-312.pyc b/__pycache__/stores.cpython-312.pyc new file mode 100644 index 0000000..1da06fb Binary files /dev/null and b/__pycache__/stores.cpython-312.pyc differ diff --git a/.gitignore b/crawler_ai_project_files/.gitignore similarity index 100% rename from .gitignore rename to crawler_ai_project_files/.gitignore diff --git a/.python-version b/crawler_ai_project_files/.python-version similarity index 100% rename from .python-version rename to crawler_ai_project_files/.python-version diff --git a/README.md b/crawler_ai_project_files/README.md similarity index 100% rename from README.md rename to crawler_ai_project_files/README.md diff --git a/config.py b/crawler_ai_project_files/config.py similarity index 100% rename from config.py rename to crawler_ai_project_files/config.py diff --git a/ingestion/adapters/__init__.py b/crawler_ai_project_files/ingestion/adapters/__init__.py similarity index 100% rename from ingestion/adapters/__init__.py rename to crawler_ai_project_files/ingestion/adapters/__init__.py diff --git a/ingestion/adapters/api_adapter.py b/crawler_ai_project_files/ingestion/adapters/api_adapter.py similarity index 100% rename from ingestion/adapters/api_adapter.py rename to crawler_ai_project_files/ingestion/adapters/api_adapter.py diff --git a/ingestion/adapters/base.py b/crawler_ai_project_files/ingestion/adapters/base.py similarity index 100% rename from ingestion/adapters/base.py rename to crawler_ai_project_files/ingestion/adapters/base.py diff --git a/ingestion/adapters/file_adapter.py b/crawler_ai_project_files/ingestion/adapters/file_adapter.py similarity index 100% rename from ingestion/adapters/file_adapter.py rename to crawler_ai_project_files/ingestion/adapters/file_adapter.py diff --git a/ingestion/adapters/web_scraper_adapter.py b/crawler_ai_project_files/ingestion/adapters/web_scraper_adapter.py similarity index 100% rename from ingestion/adapters/web_scraper_adapter.py rename to crawler_ai_project_files/ingestion/adapters/web_scraper_adapter.py diff --git a/ingestion/ingestor.py b/crawler_ai_project_files/ingestion/ingestor.py similarity index 100% rename from ingestion/ingestor.py rename to crawler_ai_project_files/ingestion/ingestor.py diff --git a/log/__init__.py b/crawler_ai_project_files/log/__init__.py similarity index 100% rename from log/__init__.py rename to crawler_ai_project_files/log/__init__.py diff --git a/log/log_stream.py b/crawler_ai_project_files/log/log_stream.py similarity index 100% rename from log/log_stream.py rename to crawler_ai_project_files/log/log_stream.py diff --git a/log/logging_utils.py b/crawler_ai_project_files/log/logging_utils.py similarity index 100% rename from log/logging_utils.py rename to crawler_ai_project_files/log/logging_utils.py diff --git a/main.py b/crawler_ai_project_files/main.py similarity index 100% rename from main.py rename to crawler_ai_project_files/main.py diff --git a/models.py b/crawler_ai_project_files/models.py similarity index 100% rename from models.py rename to crawler_ai_project_files/models.py diff --git a/normalization/__init__.py b/crawler_ai_project_files/normalization/__init__.py similarity index 100% rename from normalization/__init__.py rename to crawler_ai_project_files/normalization/__init__.py diff --git a/normalization/base.py b/crawler_ai_project_files/normalization/base.py similarity index 100% rename from normalization/base.py rename to crawler_ai_project_files/normalization/base.py diff --git a/normalization/normalizer.py b/crawler_ai_project_files/normalization/normalizer.py similarity index 100% rename from normalization/normalizer.py rename to crawler_ai_project_files/normalization/normalizer.py diff --git a/normalization/utils.py b/crawler_ai_project_files/normalization/utils.py similarity index 100% rename from normalization/utils.py rename to crawler_ai_project_files/normalization/utils.py diff --git a/pyproject.toml b/crawler_ai_project_files/pyproject.toml similarity index 100% rename from pyproject.toml rename to crawler_ai_project_files/pyproject.toml diff --git a/data/sample.json b/crawler_ai_project_files/sample.json similarity index 100% rename from data/sample.json rename to crawler_ai_project_files/sample.json diff --git a/schema.yaml b/crawler_ai_project_files/schema.yaml similarity index 100% rename from schema.yaml rename to crawler_ai_project_files/schema.yaml diff --git a/services.py b/crawler_ai_project_files/services.py similarity index 100% rename from services.py rename to crawler_ai_project_files/services.py diff --git a/stores.py b/crawler_ai_project_files/stores.py similarity index 100% rename from stores.py rename to crawler_ai_project_files/stores.py diff --git a/crawler_ai_project_files/testfile b/crawler_ai_project_files/testfile new file mode 100644 index 0000000..887d41b --- /dev/null +++ b/crawler_ai_project_files/testfile @@ -0,0 +1,28 @@ +{ + "name": "Demo Data Pipeline", + "sources": [ + { + "type": "api", + "config": { + "url": "https://dummyjson.com/posts" + } + }, + { + "type": "scrape", + "config": { + "urls": [ + "https://www.thairath.co.th/home", + "https://www.thaipbs.or.th/news" + ], + "prompt": "Extract all post in the pages along with its brief details and url" + } + }, + { + "type": "file", + "config": { + "path": "/data/sample.json", + "format": "json" + } + } + ] +} \ No newline at end of file diff --git a/tests/__init__.py b/crawler_ai_project_files/tests/__init__.py similarity index 100% rename from tests/__init__.py rename to crawler_ai_project_files/tests/__init__.py diff --git a/tests/test_api_adapter.py b/crawler_ai_project_files/tests/test_api_adapter.py similarity index 100% rename from tests/test_api_adapter.py rename to crawler_ai_project_files/tests/test_api_adapter.py diff --git a/uv.lock b/crawler_ai_project_files/uv.lock similarity index 100% rename from uv.lock rename to crawler_ai_project_files/uv.lock