granicus_archiver.config

granicus_archiver.config.GroupKey

alias of Literal[‘root’, ‘google’, ‘aws’, ‘legistar’]

granicus_archiver.config.get_app_config(*parts: Path | str) → Path[source]

Get the user config directory for the app

Any arguments (if provided) will be joined to the path

Parameters:

parts (Path | str)

Return type:

Path

granicus_archiver.config.get_app_cache(*parts: Path | str) → Path[source]

Get the user cache directory for the app

Any arguments (if provided) will be joined to the path

Parameters:

parts (Path | str)

Return type:

Path

class granicus_archiver.config.BaseConfig[source]

Bases: Serializable

group_key: ClassVar[Literal['root', 'google', 'aws', 'legistar']]

Unique key for BaseConfig subclasses

classmethod iter_child_config_classes() → Iterator[tuple[str, type[BaseConfig]]][source]

Iterate over child config classes

Return type:

Iterator[tuple[str, type[BaseConfig]]]

property child_configs: dict[str, BaseConfig]

Mapping of child config instances by their attribute name

abstract update(**kwargs) → bool[source]

Update the config from keyword arguments

Return type:

bool

abstract classmethod build_defaults(**kwargs) → Self[source]

Create the config using defaults

Any provided keyword arguments will override the default

Return type:

Self

abstract classmethod load_from_env() → Self[source]

Load the config from environment variables

Return type:

Self

as_dotenv() → str[source]

Convert the config to a dotenv string

Return type:

str

class granicus_archiver.config.GoogleConfig(user_credentials_filename: Path, drive_folder: Path, legistar_drive_folder: Path, rguid_legistar_drive_folder: Path, folder_cache_file: Path, meta_cache_file: Path)[source]

Bases: BaseConfig

Google config

Parameters:

  • user_credentials_filename (Path)

  • drive_folder (Path)

  • legistar_drive_folder (Path)

  • rguid_legistar_drive_folder (Path)

  • folder_cache_file (Path)

  • meta_cache_file (Path)

user_credentials_filename: Path

Path to store OAuth credentials

drive_folder: Path

Root folder name to upload within Drive

legistar_drive_folder: Path

Root folder name to upload legistar items within Drive

rguid_legistar_drive_folder: Path

Root folder name to upload rguid legistar items within Drive

folder_cache_file: Path

Path to store folder cache

meta_cache_file: Path

Path to store metadata cache

group_key: ClassVar[Literal['root', 'google', 'aws', 'legistar']] = 'google'

Unique key for BaseConfig subclasses

update(**kwargs) → bool[source]

Update the config from keyword arguments

Return type:

bool

classmethod build_defaults(**kwargs) → Self[source]

Create the config using defaults

Any provided keyword arguments will override the default

Return type:

Self

classmethod load_from_env() → Self[source]

Load the config from environment variables

Return type:

Self

class granicus_archiver.config.AWSConfig(bucket_name: str, clips_prefix: Path, legistar_prefix: Path, legistar_rguid_prefix: Path, region_name: str | None = None, s3_endpoint_url: URL | None = None, credentials_profile: str = 'default', access_key_id: str | None = None, secret_access_key: str | None = None, object_url_format: str = 'https://s3.amazonaws.com/{bucket_name}/{key}')[source]

Bases: BaseConfig

AWS Config

Parameters:

  • bucket_name (str)

  • clips_prefix (Path)

  • legistar_prefix (Path)

  • legistar_rguid_prefix (Path)

  • region_name (str | None)

  • s3_endpoint_url (URL | None)

  • credentials_profile (str)

  • access_key_id (str | None)

  • secret_access_key (str | None)

  • object_url_format (str)

default_object_url_fmt: ClassVar[str] = 'https://s3.amazonaws.com/{bucket_name}/{key}'

Default object_url_format

bucket_name: str

The bucket to use for the archive

clips_prefix: Path

Prefix for clips

legistar_prefix: Path

Prefix for legistar items

legistar_rguid_prefix: Path

Prefix for rguid legistar items

region_name: str | None = None

AWS region name. If not set, the default region will be used

s3_endpoint_url: URL | None = None

AWS S3 endpoint URL. If not set, the default endpoint will be used

credentials_profile: str = 'default'

The AWS credentials profile to use (from ~/.aws/credentials)

access_key_id: str | None = None

AWS Access Key ID. If not set, the default credentials provider chain will be used

secret_access_key: str | None = None

AWS Secret Access Key. If not set, the default credentials provider chain will be used

object_url_format: str = 'https://s3.amazonaws.com/{bucket_name}/{key}'

Format string for generating object URLs.

Required fields are

group_key: ClassVar[Literal['root', 'google', 'aws', 'legistar']] = 'aws'

Unique key for BaseConfig subclasses

update(**kwargs) → bool[source]

Update the config from keyword arguments

Return type:

bool

get_object_url(key: Key, scheme: str | None = None) → URL[source]

Get a URL for an S3 key within bucket_name

Parameters:

  • key (Key)

  • scheme (str | None)

Return type:

URL

classmethod build_defaults(**kwargs) → Self[source]

Create the config using defaults

Any provided keyword arguments will override the default

Return type:

Self

classmethod load_from_env() → Self[source]

Load the config from environment variables

Return type:

Self

class granicus_archiver.config.LegistarConfig(out_dir: ~pathlib._local.Path, out_dir_abs: ~pathlib._local.Path, data_file: ~pathlib._local.Path, search_index_dir: ~pathlib._local.Path, feed_urls: dict[str, ~yarl.URL] = <factory>, feed_overflows_allowed: list[str] = <factory>, category_maps: dict[str, str] = <factory>)[source]

Bases: BaseConfig

Legistar Config

Parameters:

  • out_dir (Path)

  • out_dir_abs (Path)

  • data_file (Path)

  • search_index_dir (Path)

  • feed_urls (dict[str, URL])

  • feed_overflows_allowed (list[str])

  • category_maps (dict[str, str])

out_dir: Path

Root directory to store downloaded files (relatve to the current working directory). Defaults to data/legistar

out_dir_abs: Path

Root directory to store downloaded files (absolute path)

data_file: Path

Filename to store parsed data. Defaults to “<out-dir>/data.json”

search_index_dir: Path

Directory to store the search index (relative to the current working directory)

feed_urls: dict[str, URL]

Mapping of calendar RSS feed urls with user-defined names as keys

feed_overflows_allowed: list[str]

A list of feed names (keys of feed_urls) that are allowed to reach the 100 item limit described in legistar.rss_parser.Feed.from_feed()

category_maps: dict[str, str]

A dict of any custom mappings to match the Clip.location fields to their appropriate legistar.rss_parser.FeedItem.category

The keys for this should be the location with the values set to the category.

group_key: ClassVar[Literal['root', 'google', 'aws', 'legistar']] = 'legistar'

Unique key for BaseConfig subclasses

is_feed_overflow_allowed(feed: str | URL) → bool[source]

Check whether the given feed name or url is allowed to overflow

If a string is supplied this returns whether it is present in feed_overflows_allowed.

If a URL is supplied, feed_urls will be searched and the matching key (if any) will be checked.

Parameters:

feed (str | URL)

Return type:

bool

update(**kwargs) → bool[source]

Update the config from keyword arguments

Return type:

bool

classmethod build_defaults(**kwargs) → Self[source]

Create the config using defaults

Any provided keyword arguments will override the default

Return type:

Self

classmethod load_from_env() → Self[source]

Load the config from environment variables

Return type:

Self

class granicus_archiver.config.Config(out_dir: 'Path', out_dir_abs: 'Path', data_file: 'Path', timestamp_file: 'Path', granicus_data_url: 'URL | None', legistar: 'LegistarConfig', google: 'GoogleConfig', aws: 'AWSConfig', local_timezone_name: 'str | None')[source]

Bases: BaseConfig

Parameters:
out_dir: Path

Root directory to store downloaded files (relatve to the current working directory)

out_dir_abs: Path

Root directory to store downloaded files (absolute path)

data_file: Path

Filename to store download information. Defaults to “<out-dir>/data.json”

timestamp_file: Path

Filename to store clip timestamp information. Defaults to “<out-dir>/timestamp-data.yaml”

granicus_data_url: URL | None

URL for granicus clip data

legistar: LegistarConfig

LegistarConfig instance

google: GoogleConfig

GoogleConfig instance

aws: AWSConfig

AWSConfig instance

local_timezone_name: str | None

Local timezone name for all granicus / legistar items

group_key: ClassVar[Literal['root', 'google', 'aws', 'legistar']] = 'root'

Unique key for BaseConfig subclasses

classmethod get_app_config(*parts: Path | str) → Path[source]

Get the user config directory for the app

Any arguments (if provided) will be joined to the path

Parameters:

parts (Path | str)

Return type:

Path

classmethod get_app_cache(*parts: Path | str) → Path[source]

Get the user cache directory for the app

Any arguments (if provided) will be joined to the path

Parameters:

parts (Path | str)

Return type:

Path

get_group(key: Literal['root', 'google', 'aws', 'legistar']) → BaseConfig[source]

Get a BaseConfig instance by its group_key

Parameters:

key (Literal['root', 'google', 'aws', 'legistar'])

Return type:

BaseConfig

classmethod load(filename: PathLike) → Self[source]

Load the config from the given filename

Parameters:

filename (PathLike)

Return type:

Self

save(filename: PathLike) → None[source]

Save the config to the given filename

Parameters:

filename (PathLike)

Return type:

None

update(**kwargs) → bool[source]

Update the config from keyword arguments

Return type:

bool

classmethod build_defaults(**kwargs) → Self[source]

Create the config using defaults

Any provided keyword arguments will override the default

Return type:

Self

classmethod load_from_env() → Self[source]

Load the config from environment variables

Return type:

Self