fix project_slug error.
replace ml_pipeline with project_name or module_name from config
This commit is contained in:
parent
727f16df57
commit
6eed08d1ba
|
@ -4,5 +4,8 @@
|
|||
"module_name": "{{ cookiecutter.repo_name }}",
|
||||
"author_name": "Your name (or your organization/company/team)",
|
||||
"description": "A short description of the project.",
|
||||
"open_source_license": ["MIT", "BSD-3-Clause", "No license file"]
|
||||
"open_source_license": ["MIT", "BSD-3-Clause", "No license file"],
|
||||
"_copy_without_render": [
|
||||
"docs/book"
|
||||
]
|
||||
}
|
||||
|
|
|
@ -0,0 +1,7 @@
|
|||
MODEL__IN_CHANNELS=1
|
||||
MODEL__NUM_CLASSES=10
|
||||
DATA__TRAIN_PATH=/path/to/{{cookiecutter.project_name}}/data/mnist_train.csv
|
||||
DATA__TEST_PATH=/path/to/{{cookiecutter.project_name}}/data/mnist_test.csv
|
||||
PATHS__APP=/path/to/{{cookiecutter.project_name}}/{{cookiecutter.module_name}}/
|
||||
PATHS__ROOT=/path/to/{{cookiecutter.project_name}}/
|
||||
PATHS__DATA=/path/to/{{cookiecutter.project_name}}/data/
|
|
@ -1,30 +1,36 @@
|
|||
APP_NAME=ml_pipeline
|
||||
APP_NAME={{cookiecutter.module_name}}
|
||||
PYTHON=.venv/bin/python3
|
||||
INTERPRETER=/usr/bin/python3
|
||||
.PHONY: help test
|
||||
|
||||
|
||||
all: run
|
||||
all: help
|
||||
|
||||
init: ## create a venv
|
||||
$(INTERPRETER) -m venv .venv
|
||||
|
||||
run: ## run the pipeline (train)
|
||||
$(PYTHON) -m $(APP_NAME) pipeline:train
|
||||
|
||||
data: ## download the mnist data
|
||||
$(PYTHON) -m $(APP_NAME) data:download
|
||||
# wget https://pjreddie.com/media/files/mnist_train.csv -O data/mnist_train.csv
|
||||
# wget https://pjreddie.com/media/files/mnist_test.csv -O data/mnist_test.csv
|
||||
|
||||
test:
|
||||
find . -iname "*.py" | entr -c pytest
|
||||
|
||||
serve:
|
||||
$(PYTHON) -m $(APP_NAME) app:serve
|
||||
cp .env.example .env
|
||||
|
||||
install:
|
||||
$(PYTHON) -m pip install -r requirements.txt
|
||||
|
||||
data: ## download the mnist data
|
||||
$(PYTHON) -m $(APP_NAME) data:download
|
||||
|
||||
run: ## run the pipeline (train)
|
||||
$(PYTHON) -m $(APP_NAME) pipeline:train
|
||||
|
||||
serve: ## start fastapi uvicorn server
|
||||
$(PYTHON) -m $(APP_NAME) app:serve
|
||||
|
||||
docs: ## serve the mdbook docs directory
|
||||
mdbook serve docs
|
||||
|
||||
test: ## run pytest tests
|
||||
$(PYTHON) -m pytest
|
||||
|
||||
test-watch: ## run pytest on .py changes
|
||||
find . -iname "*.py" | entr -c $(PYTHON) -m pytest
|
||||
|
||||
help: ## display this help message
|
||||
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
|
||||
@echo "available commands:"
|
||||
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36mmake %-30s\033[0m %s\n", $$1, $$2}'
|
|
@ -3,4 +3,4 @@ authors = ["publicmatt"]
|
|||
language = "en"
|
||||
multilingual = false
|
||||
src = "src"
|
||||
title = "ml_pipeline"
|
||||
title = "{{cookiecutter.project_name}}"
|
|
@ -3,7 +3,7 @@ requires = ["setuptools", "wheel"]
|
|||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "ml_pipeline"
|
||||
name = "{{cookiecutter.project_name}}"
|
||||
version = "0.1.0"
|
||||
authors = [
|
||||
{name = "publicmatt", email = "git@publicmatt.com"},
|
||||
|
@ -18,7 +18,6 @@ dependencies = [
|
|||
"numpy==1.26.4",
|
||||
"pytest==8.1.1",
|
||||
"pytest-cov==5.0.0",
|
||||
"python-dotenv==1.0.1",
|
||||
"requests==2.31.0",
|
||||
"torch==2.2.2",
|
||||
"torchvision=0.17.2",
|
||||
|
@ -37,11 +36,11 @@ repository = "https://example.com/my_project/repo"
|
|||
documentation = "https://example.com/my_project/docs"
|
||||
|
||||
[tool.setuptools]
|
||||
packages = ["ml_pipeline"]
|
||||
packages = ["{{cookiecutter.module_name}}"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
# Run tests in parallel using pytest-xdist
|
||||
addopts = "--cov=ml_pipeline --cov-report=term"
|
||||
addopts = "--cov={{cookiecutter.module_name}} --cov-report=term"
|
||||
# Specify the paths to look for tests
|
||||
testpaths = [
|
||||
"test",
|
|
@ -0,0 +1,6 @@
|
|||
from {{cookiecutter.module_name}} import config
|
||||
from {{cookiecutter.module_name}}.model.cnn import VGG11
|
||||
|
||||
def test_in_channels():
|
||||
assert config.model.name == 'vgg11'
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
from ml_pipeline.data.dataset import MnistDataset
|
||||
from ml_pipeline import config
|
||||
from {{cookiecutter.module_name}}.data.dataset import MnistDataset
|
||||
from {{cookiecutter.module_name}} import config
|
||||
from pathlib import Path
|
||||
import pytest
|
||||
|
|
@ -5,7 +5,7 @@ pwd = Path(__file__).parent
|
|||
config_path = pwd / 'config'
|
||||
root_path = pwd.parent
|
||||
config = ConfigurationSet(
|
||||
config_from_env(prefix="ML_PIPELINE", separator="__", lowercase_keys=True),
|
||||
config_from_env(prefix="{{cookiecutter.module_name.upper()}}", separator="__", lowercase_keys=True),
|
||||
config_from_dotenv(root_path / ".env", read_from_file=True, lowercase_keys=True, interpolate=True, interpolate_type=1),
|
||||
config_from_toml(config_path / "training.toml", read_from_file=True),
|
||||
config_from_toml(config_path / "data.toml", read_from_file=True),
|
|
@ -0,0 +1,5 @@
|
|||
from {{cookiecutter.module_name}}.cli import cli
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
from {{cookiecutter.module_name}} import config
|
||||
from fastapi import FastAPI, Response
|
||||
import logging
|
||||
import uvicorn
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def run():
|
||||
uvicorn.run("{{cookiecutter.module_name}}.app:app", host=config.app.host, port=config.app.port, proxy_headers=True)
|
|
@ -4,38 +4,38 @@ import click
|
|||
@click.version_option()
|
||||
def cli():
|
||||
"""
|
||||
ml_pipeline: a template for building, training and running pytorch models.
|
||||
build, train and run machine learning models.
|
||||
"""
|
||||
|
||||
|
||||
@cli.command("pipeline:train")
|
||||
def pipeline_train():
|
||||
"""run the training pipeline with train data"""
|
||||
from ml_pipeline.training import pipeline
|
||||
from {{cookiecutter.module_name}}.training import pipeline
|
||||
pipeline.run(evaluate=False)
|
||||
|
||||
@cli.command("pipeline:evaluate")
|
||||
def pipeline_evaluate():
|
||||
"""run the training pipeline with test data"""
|
||||
from ml_pipeline.training import pipeline
|
||||
from {{cookiecutter.module_name}}.training import pipeline
|
||||
pipeline.run(evaluate=True)
|
||||
|
||||
@cli.command("app:serve")
|
||||
def app_serve():
|
||||
"""run the api server pipeline with pretrained model"""
|
||||
from ml_pipeline import app
|
||||
from {{cookiecutter.module_name}} import app
|
||||
app.run()
|
||||
|
||||
@cli.command("data:download")
|
||||
def data_download():
|
||||
"""download the train and test data"""
|
||||
from ml_pipeline import data
|
||||
from ml_pipeline import config
|
||||
from {{cookiecutter.module_name}} import data
|
||||
from {{cookiecutter.module_name}} import config
|
||||
from pathlib import Path
|
||||
data.download(Path(config.paths.data))
|
||||
|
||||
@cli.command("data:debug")
|
||||
def data_debug():
|
||||
"""debug the dataset class"""
|
||||
from ml_pipeline.data import dataset
|
||||
from {{cookiecutter.module_name}}.data import dataset
|
||||
dataset.debug()
|
|
@ -1,4 +1,4 @@
|
|||
[paths]
|
||||
repo = "/path/to/root"
|
||||
app = "/path/to/root/ml_pipeline"
|
||||
app = "/path/to/root/{{cookiecutter.module_name}}"
|
||||
data = "/path/to/root/data"
|
|
@ -1,7 +1,7 @@
|
|||
from pathlib import Path
|
||||
import requests
|
||||
import logging
|
||||
from ml_pipeline import config
|
||||
from {{cookiecutter.module_name}} import config
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
|
@ -5,7 +5,7 @@ import csv
|
|||
import torch
|
||||
from pathlib import Path
|
||||
from typing import Tuple
|
||||
from ml_pipeline import config, logger
|
||||
from {{cookiecutter.module_name}} import config, logger
|
||||
|
||||
|
||||
class MnistDataset(Dataset):
|
|
@ -8,7 +8,7 @@
|
|||
"outputs": [],
|
||||
"source": [
|
||||
"# Now you can import your package\n",
|
||||
"import ml_pipeline"
|
||||
"import {{cookiecutter.module_name}}"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
@ -18,7 +18,7 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from ml_pipeline.data.dataset import MnistDataset"
|
||||
"from {{cookiecutter.module_name}}.data.dataset import MnistDataset"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
@ -28,38 +28,9 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from ml_pipeline import config"
|
||||
"from {{cookiecutter.module_name}} import config"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "c8ce7920-c056-44ac-93df-b25bae870592",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<ConfigurationSet: 0x7fcf70fc1a50>"
|
||||
]
|
||||
},
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"config"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "83293ef7-37b3-452f-8de5-13bee633d099",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
|
@ -1,8 +1,8 @@
|
|||
|
||||
from torch.utils.data import DataLoader
|
||||
from torch.optim import AdamW
|
||||
from ml_pipeline.training.runner import Runner
|
||||
from ml_pipeline import config, logger
|
||||
from {{cookiecutter.module_name}}.training.runner import Runner
|
||||
from {{cookiecutter.module_name}} import config, logger
|
||||
|
||||
|
||||
def run(evaluate=False):
|
||||
|
@ -30,8 +30,8 @@ def run(evaluate=False):
|
|||
logger.info(f"{step}")
|
||||
|
||||
def get_model(name='vgg11'):
|
||||
from ml_pipeline.model.linear import DNN
|
||||
from ml_pipeline.model.cnn import VGG11
|
||||
from {{cookiecutter.module_name}}.model.linear import DNN
|
||||
from {{cookiecutter.module_name}}.model.cnn import VGG11
|
||||
if name == 'vgg11':
|
||||
return VGG11(config.data.in_channels, config.data.num_classes)
|
||||
else:
|
||||
|
@ -43,7 +43,7 @@ def get_model(name='vgg11'):
|
|||
|
||||
def get_dataset(evaluate=False):
|
||||
# Usage
|
||||
from ml_pipeline.data.dataset import MnistDataset
|
||||
from {{cookiecutter.module_name}}.data.dataset import MnistDataset
|
||||
from torchvision import transforms
|
||||
csv_file_path = config.data.train_path if not evaluate else config.data.test_path
|
||||
transform = transforms.Compose([
|
|
@ -1,6 +0,0 @@
|
|||
from ml_pipeline import config
|
||||
from ml_pipeline.model.cnn import VGG11
|
||||
|
||||
def test_in_channels():
|
||||
assert config.model.name == 'vgg11'
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
from ml_pipeline.cli import cli
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
from ml_pipeline import config
|
||||
from fastapi import FastAPI, Response
|
||||
import logging
|
||||
import uvicorn
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def run():
|
||||
uvicorn.run("ml_pipeline.app:app", host=config.app.host, port=config.app.port, proxy_headers=True)
|
Loading…
Reference in New Issue