Compare commits

...
2 Commits
26 changed files with 52210 additions and 0 deletions
+8
View File
@@ -0,0 +1,8 @@
# PostgreSQL Database Configuration
# Copy this file to .env and fill in your actual values
POSTGRES_HOST=localhost
POSTGRES_PORT=5432
POSTGRES_DB=home_assistant
POSTGRES_USER=your_username
POSTGRES_PASSWORD=your_password
+56
View File
@@ -0,0 +1,56 @@
# Python
__pycache__/
*.py[cod]
*$py.class
*.so
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
# Virtual environments
.venv/
venv/
ENV/
env/
# Jupyter Notebook
.ipynb_checkpoints/
*.ipynb_checkpoints
# Environment variables
.env
.env.local
# Database
*.db
*.sqlite
*.sqlite3
# IDEs
.vscode/
.idea/
*.swp
*.swo
*~
# OS
.DS_Store
Thumbs.db
# Project specific
data/
*.csv
*.parquet
+135
View File
@@ -0,0 +1,135 @@
# Home Assistant Data Analysis
Data analysis notebooks for investigating power consumption and other smart home metrics collected by Home Assistant.
## Project Structure
```bash
.
├── .gitea/
│ └── workflows/
│ └── setup_database.yml # CI workflow to setup PostgreSQL database
├── notebooks/
│ └── boiler_warning.ipynb # Power consumption analysis notebook
├── utils/
│ ├── __init__.py
│ └── db_connection.py # Shared database connection utilities
├── .env.example # Example environment variables
├── .gitignore
├── pyproject.toml # Project dependencies and configuration
└── README.md
```
## Setup
### 1. Install UV (Python Package Manager)
```bash
# macOS/Linux
curl -LsSf https://astral.sh/uv/install.sh | sh
# Or via Homebrew
brew install uv
```
### 2. Create Virtual Environment and Install Dependencies
```bash
# Create and activate virtual environment with UV
uv venv
source .venv/bin/activate # On macOS/Linux
# Install dependencies
uv pip install -e .
```
### 3. Configure Database Connection
```bash
# Copy example environment file
cp .env.example .env
# Edit .env with your PostgreSQL credentials
nano .env # or use your preferred editor
```
### 4. Run Jupyter Notebooks
```bash
# Start Jupyter Lab
jupyter lab
# Or Jupyter Notebook
jupyter notebook
```
## Database Setup
The repository includes a CI workflow (`.gitea/workflows/setup_database.yml`) that can be manually triggered to initialize the PostgreSQL database. Ensure the following secrets are configured in your Gitea repository:
- `POSTGRES_HOST`
- `POSTGRES_PORT`
- `POSTGRES_ROOT_PASSWORD`
- `DB_NAME`
- `DB_USER`
- `DB_PASSWORD`
## Notebooks
### Current Notebooks
- **boiler_warning.ipynb**: Analysis of power consumption data from Home Assistant
### Adding New Notebooks
1. Create new notebook in the `notebooks/` directory
2. Use descriptive names (e.g., `temperature_trends.ipynb`, `energy_efficiency.ipynb`)
3. Import shared utilities: `from utils import get_db_connection, get_db_engine`
## Shared Utilities
The `utils/` module provides common functions for database connections:
```python
from utils import get_db_connection, get_db_engine
# Using psycopg2 (for raw SQL)
conn = get_db_connection()
cursor = conn.cursor()
cursor.execute("SELECT * FROM states LIMIT 10")
# Using SQLAlchemy (for pandas integration)
engine = get_db_engine()
import pandas as pd
df = pd.read_sql("SELECT * FROM states LIMIT 10", engine)
```
## Development
### Optional Development Tools
Install development dependencies for code formatting and linting:
```bash
uv pip install -e ".[dev]"
```
Format code with Black:
```bash
black utils/
```
Lint code with Ruff:
```bash
ruff check utils/
```
## Contributing
When adding new analysis notebooks:
1. Document the purpose and findings in the notebook
2. Add any new dependencies to `pyproject.toml`
3. Update this README if adding significant new functionality
File diff suppressed because one or more lines are too long
+40
View File
@@ -0,0 +1,40 @@
[project]
name = "home-assistant"
version = "0.1.0"
description = "Data analysis notebooks for Home Assistant smart home metrics"
readme = "README.md"
requires-python = ">=3.12"
dependencies = [
"jupyter>=1.0.0",
"pandas>=2.0.0",
"matplotlib>=3.7.0",
"seaborn>=0.13.0",
"psycopg2-binary>=2.9.0",
"sqlalchemy>=2.0.0",
"python-dotenv>=1.0.0",
"ipykernel>=6.25.0",
"pandera>=0.28.1",
"plotly>=6.5.2",
]
[project.optional-dependencies]
dev = [
"ruff>=0.1.0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["utils"]
[tool.ruff]
line-length = 100
target-version = "py312"
[dependency-groups]
dev = [
"pandas-stubs>=2.3.3.260113",
"ruff>=0.14.14",
]
+9
View File
@@ -0,0 +1,9 @@
"""Shared utilities for Home Assistant data analysis."""
from .get_db_session import get_db_session
from .get_state import get_state
__all__ = [
"get_db_session",
"get_state",
]
+7
View File
@@ -0,0 +1,7 @@
"""Definition of dataframes."""
from .sensor_state import SensorStateSchema
__all__ = [
"SensorStateSchema",
]
+12
View File
@@ -0,0 +1,12 @@
"""Definition of SensorStateSchema dataframe schema."""
from typing import Optional
import pandera.pandas as pa
import pandas as pd
class SensorStateSchema(pa.DataFrameModel):
"""Schema for sensor state dataframe."""
time: pd.DatetimeTZDtype = pa.Field(dtype_kwargs={"tz": "UTC"})
state: Optional[str]
+61
View File
@@ -0,0 +1,61 @@
"""Database connection utilities for Home Assistant PostgreSQL database."""
import os
from functools import lru_cache
from dotenv import load_dotenv
from sqlalchemy import create_engine, text
from sqlalchemy.orm import sessionmaker
# Load environment variables from .env file
load_dotenv()
@lru_cache(maxsize=1)
def _get_engine(echo: bool = False):
"""Create and cache the database engine (singleton)."""
required_vars = [
"POSTGRES_HOST",
"POSTGRES_PORT",
"POSTGRES_DB",
"POSTGRES_USER",
"POSTGRES_PASSWORD",
]
missing_vars = [var for var in required_vars if not os.getenv(var)]
if missing_vars:
raise ValueError(f"Missing required environment variables: {', '.join(missing_vars)}")
connection_string = (
f"postgresql://{os.getenv('POSTGRES_USER')}:{os.getenv('POSTGRES_PASSWORD')}"
f"@{os.getenv('POSTGRES_HOST')}:{os.getenv('POSTGRES_PORT')}/{os.getenv('POSTGRES_DB')}"
)
return create_engine(connection_string, echo=echo)
def get_db_session(echo: bool = False) -> sessionmaker:
"""
Create a SQLAlchemy sessionmaker for the Home Assistant PostgreSQL database.
Args:
echo: If True, log all SQL statements (default: False)
Returns:
sessionmaker: Database session maker object
Raises:
ValueError: If required environment variables are not set
"""
# Create the SQLAlchemy engine
engine = _get_engine(echo=echo)
# Create a session maker
session = sessionmaker(bind=engine)
# Test connection to database
with session.begin() as s:
s.execute(text("SELECT 1"))
return session
+89
View File
@@ -0,0 +1,89 @@
"""Definition of get_state function."""
from datetime import datetime, UTC
from pandera.typing import DataFrame
import pandas as pd
import pandera.pandas as pa
import sqlalchemy as sa
from utils.database.tables import (
States,
StatesMeta,
)
from utils.database.dataframes import SensorStateSchema
from utils.database import get_db_session
@pa.check_types
def get_state(
entity_id: str,
limit: int | None = None,
start_time: datetime | None = None,
end_time: datetime | None = None,
) -> DataFrame[SensorStateSchema]:
"""
Retrieve sensor state data.
Args:
entity_id: The entity ID of the sensor.
limit: Number of recent records to retrieve (default is None for all records).
start_time: Start of time range (inclusive). If None, no lower bound.
end_time: End of time range (inclusive). If None, no upper bound.
Returns:
DataFrame[SensorStateSchema]: The retrieved sensor state.
"""
# Create a new database session
session = get_db_session()
# Prepare the base statement
stmt = (
sa.select(
States.state,
sa.func.to_timestamp(States.last_updated_ts).label("time"),
)
.join(
StatesMeta,
States.metadata_id == StatesMeta.metadata_id,
)
.where(StatesMeta.entity_id == entity_id)
)
# Add time range filters if provided
if start_time is not None:
start_ts = start_time.astimezone(UTC).timestamp()
stmt = stmt.where(States.last_updated_ts >= start_ts)
if end_time is not None:
end_ts = end_time.astimezone(UTC).timestamp()
stmt = stmt.where(States.last_updated_ts <= end_ts)
# Order by time descending and apply limit if specified
stmt = stmt.order_by(States.last_updated_ts.desc())
if limit is not None:
stmt = stmt.limit(limit)
# Execute query
with session.begin() as s:
df = pd.read_sql(stmt, s.connection())
# Handle empty results
if len(df) == 0:
# Create empty DataFrame with correct schema
empty_df = pd.DataFrame(
{
"time": pd.Series([], dtype="datetime64[ns, UTC]"),
"state": pd.Series([], dtype="object"),
}
)
return DataFrame[SensorStateSchema](empty_df)
# Reorder columns to match schema definition
df = df[["time", "state"]]
# Convert time column to nanosecond precision (PostgreSQL returns microseconds)
df["time"] = df["time"].dt.as_unit("ns")
return DataFrame[SensorStateSchema](df)
+31
View File
@@ -0,0 +1,31 @@
"""Database table definitions for Home Assistant PostgreSQL database."""
from .event_data import EventData
from .event_types import EventTypes
from .events import Events
from .migration_changes import MigrationChanges
from .recorder_runs import RecorderRuns
from .schema_changes import SchemaChanges
from .state_attributes import StateAttributes
from .states_meta import StatesMeta
from .states import States
from .statistics_meta import StatisticsMeta
from .statistics_runs import StatisticsRuns
from .statistics_short_term import StatisticsShortTerm
from .statistics import Statistics
__all__ = [
"EventData",
"EventTypes",
"Events",
"MigrationChanges",
"RecorderRuns",
"SchemaChanges",
"StateAttributes",
"StatesMeta",
"States",
"StatisticsMeta",
"StatisticsRuns",
"StatisticsShortTerm",
"Statistics",
]
+5
View File
@@ -0,0 +1,5 @@
"""Base declarative class for SQLAlchemy ORM models."""
from sqlalchemy.orm import declarative_base
Base = declarative_base()
+16
View File
@@ -0,0 +1,16 @@
"""SQLAlchemy model for the 'event_data' table in the Home Assistant PostgreSQL database."""
from sqlalchemy import Column, BigInteger, Text
from .base import Base
class EventData(Base):
"""SQLAlchemy model for the 'event_data' table."""
__tablename__ = "event_data"
__table_args__ = {"schema": "public"}
data_id = Column(BigInteger, primary_key=True, autoincrement=True)
hash = Column(BigInteger, nullable=True)
shared_data = Column(Text, nullable=True)
+15
View File
@@ -0,0 +1,15 @@
"""SQLAlchemy model for the 'event_types' table in the Home Assistant PostgreSQL database."""
from sqlalchemy import Column, BigInteger, String
from .base import Base
class EventTypes(Base):
"""SQLAlchemy model for the 'event_types' table."""
__tablename__ = "event_types"
__table_args__ = {"schema": "public"}
event_type_id = Column(BigInteger, primary_key=True, autoincrement=True)
event_type = Column(String(64), nullable=True)
+39
View File
@@ -0,0 +1,39 @@
"""SQLAlchemy model for the 'events' table in the Home Assistant PostgreSQL database."""
from sqlalchemy import (
Column,
BigInteger,
CHAR,
SmallInteger,
TIMESTAMP,
Float,
LargeBinary,
ForeignKey,
)
from .base import Base
class Events(Base):
"""SQLAlchemy model for the 'events' table."""
__tablename__ = "events"
__table_args__ = {"schema": "public"}
event_id = Column(BigInteger, primary_key=True, autoincrement=True)
event_type = Column(CHAR(1), nullable=True)
event_data = Column(CHAR(1), nullable=True)
origin = Column(CHAR(1), nullable=True)
origin_idx = Column(SmallInteger, nullable=True)
time_fired = Column(TIMESTAMP(timezone=True), nullable=True)
time_fired_ts = Column(Float, nullable=True)
context_id = Column(CHAR(1), nullable=True)
context_user_id = Column(CHAR(1), nullable=True)
context_parent_id = Column(CHAR(1), nullable=True)
data_id = Column(BigInteger, ForeignKey("public.event_data.data_id"), nullable=True)
context_id_bin = Column(LargeBinary, nullable=True)
context_user_id_bin = Column(LargeBinary, nullable=True)
context_parent_id_bin = Column(LargeBinary, nullable=True)
event_type_id = Column(
BigInteger, ForeignKey("public.event_types.event_type_id"), nullable=True
)
@@ -0,0 +1,15 @@
"""SQLAlchemy model for the 'migration_changes' table in the Home Assistant PostgreSQL database."""
from sqlalchemy import Column, String, SmallInteger
from .base import Base
class MigrationChanges(Base):
"""SQLAlchemy model for the 'migration_changes' table."""
__tablename__ = "migration_changes"
__table_args__ = {"schema": "public"}
migration_id = Column(String(255), primary_key=True)
version = Column(SmallInteger, nullable=False)
+18
View File
@@ -0,0 +1,18 @@
"""SQLAlchemy model for the 'recorder_runs' table in the Home Assistant PostgreSQL database."""
from sqlalchemy import Column, BigInteger, TIMESTAMP, Boolean
from .base import Base
class RecorderRuns(Base):
"""SQLAlchemy model for the 'recorder_runs' table."""
__tablename__ = "recorder_runs"
__table_args__ = {"schema": "public"}
run_id = Column(BigInteger, primary_key=True, autoincrement=True)
start = Column(TIMESTAMP(timezone=True), nullable=False)
end = Column(TIMESTAMP(timezone=True), nullable=True)
closed_incorrect = Column(Boolean, nullable=False)
created = Column(TIMESTAMP(timezone=True), nullable=False)
+16
View File
@@ -0,0 +1,16 @@
"""SQLAlchemy model for the 'schema_changes' table in the Home Assistant PostgreSQL database."""
from sqlalchemy import Column, BigInteger, Integer, TIMESTAMP
from .base import Base
class SchemaChanges(Base):
"""SQLAlchemy model for the 'schema_changes' table."""
__tablename__ = "schema_changes"
__table_args__ = {"schema": "public"}
change_id = Column(BigInteger, primary_key=True, autoincrement=True)
schema_version = Column(Integer, nullable=True)
changed = Column(TIMESTAMP(timezone=True), nullable=False)
+16
View File
@@ -0,0 +1,16 @@
"""SQLAlchemy model for the 'state_attributes' table in the Home Assistant PostgreSQL database."""
from sqlalchemy import Column, BigInteger, Text
from .base import Base
class StateAttributes(Base):
"""SQLAlchemy model for the 'state_attributes' table."""
__tablename__ = "state_attributes"
__table_args__ = {"schema": "public"}
attributes_id = Column(BigInteger, primary_key=True, autoincrement=True)
hash = Column(BigInteger, nullable=True)
shared_attrs = Column(Text, nullable=True)
+45
View File
@@ -0,0 +1,45 @@
"""SQLAlchemy model for the 'states' table in the Home Assistant PostgreSQL database."""
from sqlalchemy import (
Column,
BigInteger,
CHAR,
String,
SmallInteger,
TIMESTAMP,
Float,
LargeBinary,
ForeignKey,
)
from .base import Base
class States(Base):
"""SQLAlchemy model for the 'states' table."""
__tablename__ = "states"
__table_args__ = {"schema": "public"}
state_id = Column(BigInteger, primary_key=True, autoincrement=True)
entity_id = Column(CHAR(1), nullable=True)
state = Column(String(255), nullable=True)
attributes = Column(CHAR(1), nullable=True)
event_id = Column(SmallInteger, nullable=True)
last_changed = Column(TIMESTAMP(timezone=True), nullable=True)
last_changed_ts = Column(Float, nullable=True)
last_reported_ts = Column(Float, nullable=True)
last_updated = Column(TIMESTAMP(timezone=True), nullable=True)
last_updated_ts = Column(Float, nullable=True)
old_state_id = Column(BigInteger, ForeignKey("public.states.state_id"), nullable=True)
attributes_id = Column(
BigInteger, ForeignKey("public.state_attributes.attributes_id"), nullable=True
)
context_id = Column(CHAR(1), nullable=True)
context_user_id = Column(CHAR(1), nullable=True)
context_parent_id = Column(CHAR(1), nullable=True)
origin_idx = Column(SmallInteger, nullable=True)
context_id_bin = Column(LargeBinary, nullable=True)
context_user_id_bin = Column(LargeBinary, nullable=True)
context_parent_id_bin = Column(LargeBinary, nullable=True)
metadata_id = Column(BigInteger, ForeignKey("public.states_meta.metadata_id"), nullable=True)
+15
View File
@@ -0,0 +1,15 @@
"""SQLAlchemy model for the 'states_meta' table in the Home Assistant PostgreSQL database."""
from sqlalchemy import Column, BigInteger, String
from .base import Base
class StatesMeta(Base):
"""SQLAlchemy model for the 'states_meta' table."""
__tablename__ = "states_meta"
__table_args__ = {"schema": "public"}
metadata_id = Column(BigInteger, primary_key=True, autoincrement=True)
entity_id = Column(String(255), nullable=True)
+29
View File
@@ -0,0 +1,29 @@
"""SQLAlchemy model for the 'statistics' table in the Home Assistant PostgreSQL database."""
from sqlalchemy import Column, BigInteger, TIMESTAMP, Float, ForeignKey
from .base import Base
class Statistics(Base):
"""SQLAlchemy model for the 'statistics' table."""
__tablename__ = "statistics"
__table_args__ = {"schema": "public"}
id = Column(BigInteger, primary_key=True, autoincrement=True)
created = Column(TIMESTAMP(timezone=True), nullable=True)
created_ts = Column(Float, nullable=True)
metadata_id = Column(
BigInteger, ForeignKey("public.statistics_meta.id", ondelete="CASCADE"), nullable=True
)
start = Column(TIMESTAMP(timezone=True), nullable=True)
start_ts = Column(Float, nullable=True)
mean = Column(Float, nullable=True)
mean_weight = Column(Float, nullable=True)
min = Column(Float, nullable=True)
max = Column(Float, nullable=True)
last_reset = Column(TIMESTAMP(timezone=True), nullable=True)
last_reset_ts = Column(Float, nullable=True)
state = Column(Float, nullable=True)
sum = Column(Float, nullable=True)
+22
View File
@@ -0,0 +1,22 @@
"""SQLAlchemy model for the 'statistics_meta' table in the Home Assistant PostgreSQL database."""
from sqlalchemy import Column, BigInteger, String, Boolean, SmallInteger
from .base import Base
class StatisticsMeta(Base):
"""SQLAlchemy model for the 'statistics_meta' table."""
__tablename__ = "statistics_meta"
__table_args__ = {"schema": "public"}
id = Column(BigInteger, primary_key=True, autoincrement=True)
statistic_id = Column(String(255), nullable=True)
source = Column(String(32), nullable=True)
unit_of_measurement = Column(String(255), nullable=True)
unit_class = Column(String(255), nullable=True)
has_mean = Column(Boolean, nullable=True)
has_sum = Column(Boolean, nullable=True)
name = Column(String(255), nullable=True)
mean_type = Column(SmallInteger, nullable=False)
+15
View File
@@ -0,0 +1,15 @@
"""SQLAlchemy model for the 'statistics_runs' table in the Home Assistant PostgreSQL database."""
from sqlalchemy import Column, BigInteger, TIMESTAMP
from .base import Base
class StatisticsRuns(Base):
"""SQLAlchemy model for the 'statistics_runs' table."""
__tablename__ = "statistics_runs"
__table_args__ = {"schema": "public"}
run_id = Column(BigInteger, primary_key=True, autoincrement=True)
start = Column(TIMESTAMP(timezone=True), nullable=False)
@@ -0,0 +1,29 @@
"""SQLAlchemy model for the 'statistics_short_term' table in the Home Assistant PostgreSQL database."""
from sqlalchemy import Column, BigInteger, TIMESTAMP, Float, ForeignKey
from .base import Base
class StatisticsShortTerm(Base):
"""SQLAlchemy model for the 'statistics_short_term' table."""
__tablename__ = "statistics_short_term"
__table_args__ = {"schema": "public"}
id = Column(BigInteger, primary_key=True, autoincrement=True)
created = Column(TIMESTAMP(timezone=True), nullable=True)
created_ts = Column(Float, nullable=True)
metadata_id = Column(
BigInteger, ForeignKey("public.statistics_meta.id", ondelete="CASCADE"), nullable=True
)
start = Column(TIMESTAMP(timezone=True), nullable=True)
start_ts = Column(Float, nullable=True)
mean = Column(Float, nullable=True)
mean_weight = Column(Float, nullable=True)
min = Column(Float, nullable=True)
max = Column(Float, nullable=True)
last_reset = Column(TIMESTAMP(timezone=True), nullable=True)
last_reset_ts = Column(Float, nullable=True)
state = Column(Float, nullable=True)
sum = Column(Float, nullable=True)
Generated
+2397
View File
File diff suppressed because it is too large Load Diff