diff --git a/README.md b/README.md index 0e7b5b7..903cf32 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ pip install -e . #### I want to train a Flow Matching model, where can I find the training code? -We provide [training examples](examples). Under this folder, you can find synthetic data for [continuous](examples/2d_flow_matching.ipynb), [discrete](examples/2d_discrete_flow_matching.ipynb), and [Riemannian](examples/2d_riemannian_flow_matching_flat_torus.ipynb) Flow Matching. We also provide full training [examples](examples/image) (continuous and discrete) on CIFAR10 and face-blurred ImageNet, and a scalable discrete Flow Matching example for [text modeling](examples/text). +We provide [training examples](examples). Under this folder, you can find synthetic data for [continuous](examples/2d_flow_matching.ipynb), [discrete](examples/2d_discrete_flow_matching.ipynb), [multimodal](examples/2d_multimodal_flow_matching.ipynb), and [Riemannian](examples/2d_riemannian_flow_matching_flat_torus.ipynb) Flow Matching. We also provide full training [examples](examples/image) (continuous and discrete) on CIFAR10 and face-blurred ImageNet, and a scalable discrete Flow Matching example for [text modeling](examples/text). #### Do you release pre-trained models? diff --git a/docs/Makefile b/docs/Makefile index 05c2bb7..ef4da3c 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -13,6 +13,7 @@ ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) links: mkdir -p source/notebooks && ln -sfn $(ROOT_DIR)/../examples/standalone_flow_matching.ipynb source/notebooks/standalone_flow_matching.ipynb mkdir -p source/notebooks && ln -sfn $(ROOT_DIR)/../examples/2d_discrete_flow_matching.ipynb source/notebooks/2d_discrete_flow_matching.ipynb + mkdir -p source/notebooks && ln -sfn $(ROOT_DIR)/../examples/2d_multimodal_flow_matching.ipynb source/notebooks/2d_multimodal_flow_matching.ipynb mkdir -p source/notebooks && ln -sfn $(ROOT_DIR)/../examples/2d_riemannian_flow_matching_flat_torus.ipynb source/notebooks/2d_riemannian_flow_matching_flat_torus.ipynb mkdir -p source/notebooks && ln -sfn $(ROOT_DIR)/../examples/2d_riemannian_flow_matching_sphere.ipynb source/notebooks/2d_riemannian_flow_matching_sphere.ipynb ln -sfn $(ROOT_DIR)/../assets/teaser.png source/_images/teaser.png diff --git a/docs/source/_images/multimodal.png b/docs/source/_images/multimodal.png new file mode 100644 index 0000000..a475d87 Binary files /dev/null and b/docs/source/_images/multimodal.png differ diff --git a/docs/source/dummy.rst b/docs/source/dummy.rst index 820ca98..cef70b2 100644 --- a/docs/source/dummy.rst +++ b/docs/source/dummy.rst @@ -5,5 +5,6 @@ notebooks/standalone_flow_matching notebooks/2d_discrete_flow_matching + notebooks/2d_multimodal_flow_matching notebooks/2d_riemannian_flow_matching_flat_torus notebooks/2d_riemannian_flow_matching_sphere diff --git a/docs/source/flow_matching.solver.rst b/docs/source/flow_matching.solver.rst index 99b00e8..dd8ebd1 100644 --- a/docs/source/flow_matching.solver.rst +++ b/docs/source/flow_matching.solver.rst @@ -14,5 +14,6 @@ Solvers Solver ODESolver MixtureDiscreteEulerSolver + MultimodalSolver RiemannianODESolver diff --git a/docs/source/flow_matching.utils.multimodal.rst b/docs/source/flow_matching.utils.multimodal.rst new file mode 100644 index 0000000..024e132 --- /dev/null +++ b/docs/source/flow_matching.utils.multimodal.rst @@ -0,0 +1,18 @@ +``flow_matching.utils.multimodal`` +============================= + +.. currentmodule:: flow_matching.utils.multimodal + + +Flow +-------------------------------- + +Generic multimodal flow class + +.. autosummary:: + :toctree: generated + :nosignatures: + :template: classtemplate.rst + + Flow + diff --git a/docs/source/modules.rst b/docs/source/modules.rst index 093361a..c3bc258 100644 --- a/docs/source/modules.rst +++ b/docs/source/modules.rst @@ -10,3 +10,4 @@ API Reference flow_matching.solver flow_matching.utils.model_wrapper flow_matching.utils.manifolds + flow_matching.utils.multimodal diff --git a/docs/source/notebooks.rst b/docs/source/notebooks.rst index 1967e99..5800b6a 100644 --- a/docs/source/notebooks.rst +++ b/docs/source/notebooks.rst @@ -29,4 +29,10 @@ Notebooks :image: _static/riemannian_torus.png :link: notebooks/2d_riemannian_flow_matching_flat_torus.html +.. customcarditem:: + :header: Multimodal Flow Matching + :card_description: Train and sample from a 2D Multimodal Flow Matching model. + :image: _static/multimodal.png + :link: notebooks/2d_multimodal_flow_matching.html + .. customcardend:: diff --git a/examples/2d_multimodal_flow_matching.ipynb b/examples/2d_multimodal_flow_matching.ipynb new file mode 100644 index 0000000..2a31ec1 --- /dev/null +++ b/examples/2d_multimodal_flow_matching.ipynb @@ -0,0 +1,667 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "381ea8d5", + "metadata": {}, + "source": [ + "# A simple 2D Multimodal Flow Matching model" + ] + }, + { + "cell_type": "markdown", + "id": "0c0a75af", + "metadata": {}, + "source": [ + "This notebook trains and evaluates a multimodal FM model that jointly handles\n", + "a discrete modality (categorical data) and a continuous modality (real‑valued 2‑D data).\n", + "\n", + "Dataset: 2D discrete/continuous checkerboard\n", + "Model (probability denoiser/velocity): MLPs for each modality and a shared Transformer trunk" + ] + }, + { + "cell_type": "markdown", + "id": "b5c941fc", + "metadata": {}, + "source": [ + "## Imports and init device" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "e7758331", + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "\n", + "from typing import Any, Dict, List, Sequence\n", + "\n", + "# visualization\n", + "import matplotlib.pyplot as plt\n", + "import torch\n", + "from flow_matching.path import AffineProbPath, MixtureDiscreteProbPath\n", + "from flow_matching.path.scheduler import (\n", + " CondOTScheduler, # continuous scheduler (training)\n", + " PolynomialConvexScheduler, # discrete scheduler (training)\n", + ")\n", + "\n", + "# flow_matching\n", + "from flow_matching.utils.multimodal import Flow\n", + "from torch import nn, Tensor\n", + "\n", + "# To avoid meshgrid warning\n", + "import warnings\n", + "\n", + "warnings.filterwarnings(\"ignore\", category=UserWarning, module=\"torch\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "10957ca3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using MPS\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = \"cuda:0\"\n", + " print(\"Using GPU\")\n", + "elif torch.backends.mps.is_available():\n", + " device = \"mps\"\n", + " print(\"Using MPS\")\n", + "else:\n", + " device = \"cpu\"\n", + " print(\"Using CPU\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "0491f488", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(42)" + ] + }, + { + "cell_type": "markdown", + "id": "b2ff4e5f", + "metadata": {}, + "source": [ + "## Shared model" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "1321dec5", + "metadata": {}, + "outputs": [], + "source": [ + "class SharedTransformer(nn.Module):\n", + " \"\"\"\n", + " Shared Transformer trunk used by both modalities.\n", + "\n", + " Args:\n", + " hidden_dim (int): The hidden dimension of the model.\n", + " nhead (int): The number of attention heads.\n", + " num_layers (int): The number of TransformerEncoder layers.\n", + " \"\"\"\n", + "\n", + " def __init__(self, hidden_dim: int = 128, nhead: int = 4, num_layers: int = 2):\n", + " super().__init__()\n", + " encoder_layer = nn.TransformerEncoderLayer(d_model=hidden_dim, nhead=nhead)\n", + " self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)\n", + "\n", + " def forward(self, x: Tensor) -> Tensor:\n", + " \"\"\"\n", + " Forward pass through the shared Transformer.\n", + "\n", + " Args:\n", + " x (Tensor): Input tensor of shape (sequence_length, batch_size, hidden_dim).\n", + "\n", + " Returns:\n", + " Tensor: Output tensor of the same shape as input.\n", + " \"\"\"\n", + " return self.transformer(x)" + ] + }, + { + "cell_type": "markdown", + "id": "af22ef56", + "metadata": {}, + "source": [ + "## Datasets" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "b2466329", + "metadata": {}, + "outputs": [], + "source": [ + "def inf_train_gen_discrete(\n", + " n_grid_points: int = 128,\n", + " batch_size: int = 200,\n", + " device: str = \"cpu\",\n", + ") -> Tensor:\n", + " \"\"\"\n", + " Generate a batch of discrete (categorical) samples.\n", + " Returns a tensor of shape (batch, 2) with integer token IDs.\n", + "\n", + " Args:\n", + " n_grid_points (int): Number of grid points along one axis (should be divisible by 4).\n", + " batch_size (int): Number of samples to generate.\n", + " device (str): Device to place the tensor on.\n", + "\n", + " Returns:\n", + " Tensor: A tensor of shape (batch_size, 2) with integer token IDs.\n", + " \"\"\"\n", + " assert n_grid_points % 4 == 0, \"grid size must be divisible by 4\"\n", + " n_grid_points //= 4\n", + "\n", + " x1 = torch.randint(low=0, high=n_grid_points * 4, size=(batch_size,), device=device)\n", + " samples_x2 = torch.randint(\n", + " low=0, high=n_grid_points, size=(batch_size,), device=device\n", + " )\n", + "\n", + " x2 = (\n", + " samples_x2\n", + " + 2 * n_grid_points\n", + " - torch.randint(low=0, high=2, size=(batch_size,), device=device)\n", + " * 2\n", + " * n_grid_points\n", + " + (torch.floor(x1 / n_grid_points) % 2) * n_grid_points\n", + " )\n", + " return torch.stack([x1, x2], dim=1).long()\n", + "\n", + "\n", + "def inf_train_gen_continuous(batch_size: int = 200, device: str = \"cpu\") -> Tensor:\n", + " \"\"\"\n", + " Generate a batch of 2-D continuous points from a checkerboard-like distribution.\n", + " Returns a tensor of shape (batch, 2).\n", + "\n", + " Args:\n", + " batch_size (int): Number of samples to generate.\n", + " device (str): Device to place the tensor on.\n", + "\n", + " Returns:\n", + " Tensor: A tensor of shape (batch_size, 2) with continuous values.\n", + " \"\"\"\n", + " x1 = torch.rand(batch_size, device=device) * 4 - 2\n", + " x2_ = (\n", + " torch.rand(batch_size, device=device)\n", + " - torch.randint(high=2, size=(batch_size,), device=device) * 2\n", + " )\n", + " x2 = x2_ + (torch.floor(x1) % 2)\n", + " data = torch.stack([x1, x2], dim=1) / 0.45\n", + " return data.float()" + ] + }, + { + "cell_type": "markdown", + "id": "e1faf8fd", + "metadata": {}, + "source": [ + "## Unified multimodal model" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "a3517fbc", + "metadata": {}, + "outputs": [], + "source": [ + "class Swish(nn.Module):\n", + " \"\"\"Swish activation (x * sigmoid(x)).\"\"\"\n", + "\n", + " def forward(self, x: Tensor) -> Tensor:\n", + " \"\"\"Forward pass through the Swish activation.\"\"\"\n", + " return torch.sigmoid(x) * x\n", + "\n", + "\n", + "class TransformerModel(nn.Module):\n", + " \"\"\"\n", + " A unified Transformer-based model for handling multiple modalities.\n", + "\n", + " This model processes a sequence of modalities, each with its own input\n", + " and output heads, while sharing a central Transformer trunk. It is designed\n", + " to be flexible for both discrete (categorical) and continuous data types.\n", + "\n", + " Args:\n", + " shared_transformer (SharedTransformer): The shared TransformerEncoder module.\n", + " modality_configs (List[Dict[str, Any]]): A list of dictionaries, each configuring a modality.\n", + " Required keys per config:\n", + " - 'type': 'discrete' or 'continuous'.\n", + " - 'length': The sequence length for this modality's tokens.\n", + " If 'type' is 'discrete':\n", + " - 'vocab_size': The size of the vocabulary.\n", + " If 'type' is 'continuous':\n", + " - 'input_dim': The feature dimension of the continuous data.\n", + " time_dim (int): The dimension of the time embedding.\n", + " hidden_dim (int): The hidden dimension of the model and transformer.\n", + "\n", + " Raises:\n", + " ValueError: If an unknown modality type is provided.\n", + " \"\"\"\n", + "\n", + " def __init__(\n", + " self,\n", + " shared_transformer: SharedTransformer,\n", + " modality_configs: List[Dict[str, Any]],\n", + " time_dim: int = 1,\n", + " hidden_dim: int = 128,\n", + " ):\n", + " super().__init__()\n", + " self.shared = shared_transformer\n", + " self.modality_configs = modality_configs\n", + " self.seq_lengths = [config[\"length\"] for config in modality_configs]\n", + "\n", + " self.input_embedders = nn.ModuleList()\n", + " self.time_embedders = nn.ModuleList()\n", + " self.input_projectors = nn.ModuleList()\n", + " self.output_heads = nn.ModuleList()\n", + " self.activations = nn.ModuleList()\n", + "\n", + " for config in self.modality_configs:\n", + " self.time_embedders.append(nn.Linear(1, time_dim))\n", + " self.input_projectors.append(nn.Linear(hidden_dim + time_dim, hidden_dim))\n", + " self.activations.append(Swish())\n", + "\n", + " if config[\"type\"] == \"discrete\":\n", + " self.input_embedders.append(\n", + " nn.Embedding(config[\"vocab_size\"], hidden_dim)\n", + " )\n", + " self.output_heads.append(nn.Linear(hidden_dim, config[\"vocab_size\"]))\n", + " elif config[\"type\"] == \"continuous\":\n", + " self.input_embedders.append(nn.Linear(config[\"input_dim\"], hidden_dim))\n", + " self.output_heads.append(nn.Linear(hidden_dim, config[\"input_dim\"]))\n", + " else:\n", + " raise ValueError(f\"Unknown modality type: {config['type']}\")\n", + "\n", + " def forward(\n", + " self, x_modalities: Sequence[Tensor], t_modalities: Sequence[Tensor]\n", + " ) -> Sequence[Tensor]:\n", + " \"\"\"\n", + " Forward pass for multiple modalities.\n", + "\n", + " Args:\n", + " x_modalities (Sequence[Tensor]): A sequence of input tensors, one for each modality.\n", + " Shape for discrete: (batch, length)\n", + " Shape for continuous: (batch, input_dim)\n", + " t_modalities (Sequence[Tensor]): A sequence of time tensors, one for each modality.\n", + " Shape for all: (batch, 1)\n", + "\n", + " Returns:\n", + " Sequence[Tensor]: A sequence of output tensors, one for each modality.\n", + " \"\"\"\n", + " embeddings = []\n", + "\n", + " # 1. Process each modality through its specific input head\n", + " for i, (x, t, config) in enumerate(\n", + " zip(x_modalities, t_modalities, self.modality_configs)\n", + " ):\n", + " # Embed time and expand to match sequence length\n", + " t_emb = self.time_embedders[i](t.unsqueeze(-1))\n", + " t_emb = t_emb.unsqueeze(1).expand(-1, config[\"length\"], -1)\n", + "\n", + " # Embed input based on modality type\n", + " if config[\"type\"] == \"discrete\":\n", + " x_emb = self.input_embedders[i](x) # (B, length, hidden_dim)\n", + " else: # continuous\n", + " x_emb = self.input_embedders[i](x) # (B, hidden_dim)\n", + " x_emb = x_emb.unsqueeze(1) # (B, 1, hidden_dim)\n", + "\n", + " # Combine, project, and activate\n", + " combined = torch.cat([x_emb, t_emb], dim=-1)\n", + " h = self.input_projectors[i](combined)\n", + " h = self.activations[i](h)\n", + "\n", + " # Prepare for transformer (seq_len, batch, hidden_dim)\n", + " embeddings.append(h.permute(1, 0, 2))\n", + "\n", + " # 2. Concatenate all modality embeddings and pass through shared transformer\n", + " full_sequence = torch.cat(embeddings, dim=0)\n", + " transformer_out = self.shared(full_sequence)\n", + "\n", + " # 3. Split the output and process through specific output heads\n", + " output_chunks = torch.split(transformer_out, self.seq_lengths, dim=0)\n", + " results = []\n", + " for i, chunk in enumerate(output_chunks):\n", + " # (length, B, hidden_dim) -> (B, length, hidden_dim)\n", + " chunk = chunk.permute(1, 0, 2)\n", + " output = self.output_heads[i](chunk)\n", + "\n", + " # Squeeze sequence dimension if it's 1 (for continuous case)\n", + " if output.size(1) == 1:\n", + " output = output.squeeze(1)\n", + " results.append(output)\n", + "\n", + " return results" + ] + }, + { + "cell_type": "markdown", + "id": "d5378557", + "metadata": {}, + "source": [ + "## Instantiate modalities and model" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "9b0e8daa", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- General Hyperparameters -----------------------------------------\n", + "length = 2 # 2 tokens per sample\n", + "vocab_size = 128\n", + "added_token = 0 # uniform source distribution → no extra token\n", + "vocab_size += added_token\n", + "hidden_dim = 128\n", + "\n", + "# ---- Shared transformer trunk ----------------------------------------\n", + "shared_transformer = SharedTransformer(hidden_dim=hidden_dim, nhead=4, num_layers=2).to(\n", + " device\n", + ")\n", + "\n", + "# ---- Model and Path Configuration ------------------------------------\n", + "modality_configs = [\n", + " {\n", + " \"type\": \"discrete\",\n", + " \"vocab_size\": vocab_size,\n", + " \"length\": length,\n", + " },\n", + " {\n", + " \"type\": \"continuous\",\n", + " \"input_dim\": length,\n", + " \"length\": 1, # This modality is treated as a single token in the sequence\n", + " },\n", + "]\n", + "\n", + "# A unified model that handles all modalities\n", + "model = TransformerModel(\n", + " shared_transformer=shared_transformer,\n", + " modality_configs=modality_configs,\n", + " time_dim=1,\n", + " hidden_dim=hidden_dim,\n", + ").to(device)\n", + "\n", + "# Path definitions remain distinct per modality\n", + "discrete_path = MixtureDiscreteProbPath(scheduler=PolynomialConvexScheduler(n=2.0))\n", + "continuous_path = AffineProbPath(scheduler=CondOTScheduler())\n", + "\n", + "# ---- Assemble modalities dict for Flow -------------------------------\n", + "modalities = {\n", + " \"discrete\": {\n", + " \"path\": discrete_path,\n", + " # loss omitted → Flow will use MixturePathGeneralizedKL automatically\n", + " },\n", + " \"continuous\": {\n", + " \"path\": continuous_path,\n", + " # loss omitted → Flow will use MSE loss automatically\n", + " },\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "b82a25cc", + "metadata": {}, + "source": [ + "## Instantiate the multimodal Flow model" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "9f2ccedd", + "metadata": {}, + "outputs": [], + "source": [ + "flow = Flow(model=model, modalities=modalities)\n", + "\n", + "# Optimizer (optimises both modality models)\n", + "optimizer = torch.optim.Adam(flow.parameters(), lr=1e-3)" + ] + }, + { + "cell_type": "markdown", + "id": "2636f3a4", + "metadata": {}, + "source": [ + "## Training loop" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "646de9a8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| iter 3000 | 14.35 ms/step | loss 9.040 \n", + "| iter 6000 | 16.78 ms/step | loss 9.292 \n", + "| iter 9000 | 18.14 ms/step | loss 9.037 \n", + "| iter 12000 | 18.66 ms/step | loss 9.878 \n", + "| iter 15000 | 18.56 ms/step | loss 9.466 \n", + "| iter 18000 | 18.55 ms/step | loss 9.251 \n", + "| iter 21000 | 18.27 ms/step | loss 9.220 \n", + "| iter 24000 | 18.40 ms/step | loss 9.489 \n", + "| iter 27000 | 18.48 ms/step | loss 9.835 \n", + "| iter 30000 | 18.33 ms/step | loss 9.114 \n" + ] + } + ], + "source": [ + "lr = 1e-3\n", + "batch_size = 2048\n", + "iterations = 30001\n", + "print_every = 3000\n", + "epsilon = 1e-3\n", + "\n", + "source_distribution = \"uniform\" # for the discrete modality\n", + "\n", + "start_time = time.time()\n", + "for i in range(iterations):\n", + " optimizer.zero_grad()\n", + "\n", + " # ---- Discrete data -------------------------------------------------\n", + " x1_disc = inf_train_gen_discrete(\n", + " n_grid_points=vocab_size - added_token,\n", + " batch_size=batch_size,\n", + " device=device,\n", + " )\n", + " if source_distribution == \"uniform\":\n", + " x0_disc = torch.randint_like(x1_disc, high=vocab_size)\n", + " else: # mask case (not used here)\n", + " raise NotImplementedError\n", + "\n", + " # ---- Continuous data -----------------------------------------------\n", + " x1_cont = inf_train_gen_continuous(batch_size=batch_size, device=device)\n", + " x0_cont = torch.randn_like(x1_cont) # isotropic Gaussian prior\n", + "\n", + " # ---- Sample a common time tensor for both modalities ---------------\n", + " t = torch.rand(batch_size, device=device) * (1 - epsilon)\n", + "\n", + " # ---- Sample from each path to obtain x_t ---------------------------\n", + " disc_path_sample = discrete_path.sample(t=t, x_0=x0_disc, x_1=x1_disc)\n", + " cont_path_sample = continuous_path.sample(t=t, x_0=x0_cont, x_1=x1_cont)\n", + "\n", + " # ---- Build the inputs expected by Flow.training_loss -----------\n", + " x_1 = [x1_disc, x1_cont]\n", + " x_t = [disc_path_sample.x_t, cont_path_sample.x_t]\n", + " dx_t = [None, cont_path_sample.dx_t] # NOTE: dx_t is None for discrete\n", + " ts = [t] * 2 # NOTE: For now, both modalities share the same time\n", + "\n", + " # ---- Compute total loss and back‑propagate -------------------------\n", + " loss, _ = flow.training_loss(x_1=x_1, x_t=x_t, dx_t=dx_t, t=ts)\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # ---- Logging -------------------------------------------------------\n", + " if (i + 1) % print_every == 0:\n", + " elapsed = time.time() - start_time\n", + " print(\n", + " f\"| iter {i+1:6d} | {elapsed*1000/print_every:5.2f} ms/step | loss {loss.item():8.3f} \"\n", + " )\n", + " start_time = time.time()" + ] + }, + { + "cell_type": "markdown", + "id": "e87e944d", + "metadata": {}, + "source": [ + "## Sampling from the trained multimodal model" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "e2aab2d8", + "metadata": {}, + "outputs": [], + "source": [ + "x_init = [\n", + " torch.randint_like(\n", + " x1_disc, high=vocab_size\n", + " ), # discrete initial state (uniform categorical)\n", + " torch.randn_like(x1_cont), # continuous initial state (Gaussian noise)\n", + "]\n", + "\n", + "flow.eval() # switch to eval mode for sampling\n", + "samples = flow.sample(x_init=x_init, device=device, steps=1000)" + ] + }, + { + "cell_type": "markdown", + "id": "2bceb4bb", + "metadata": {}, + "source": [ + "## Visualization" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "43dc2909", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkcAAAHqCAYAAAAOKepaAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjYsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvq6yFwwAAAAlwSFlzAAAPYQAAD2EBqD+naQAAZyFJREFUeJzt3Qu8VWP6B/DnnE51Up1SuugeouhGSBhFKQo1LqP+jSjX5BIzqFySS7mGDBWGxtBERg4NuZRqUroSiYRGzXQdqVS6nHPW//N7zd7W3mev3bvXZa+11/p9fZbOXnuv615r73e/z/u8b55hGIYQERERkZL/yz9EREREBCwcEREREZmwcERERERkwsIRERERkQkLR0REREQmLBwRERERmbBwRERERGTCwhERERGRCQtHRERERCYsHAXA3XffLXl5eX7vRuTgnOPcZ2r27NlqWfwbc9lll0mzZs1c3sPw6NKli5qybefOnVK3bl15+eWXXV0v3v/rrrtOggDX3TnnnGPruqXyhg0bJh07dvR7N8hnLBy5bNKkSeoDKDYVFhZKgwYNpEePHjJu3Dj56aefJJesXLlSFSD+9a9/+b0rOWX37t3qvPGLyF9PPPGEVK9eXfr27Ruf9/bbb9sqFEfV6NGj5Y033pCoGDp0qCxfvlzefPNNv3eFfMTCkUfuuece+etf/yrjx4+X66+/Pn7TtWnTRj777LOE195xxx3y888/S1ALR6NGjWLh6ACeffZZWbVqVULhCOeNhSP/7N+/XxWOrrjiCqlQoUJC4QjvTdScdtpp6nMG/2YiaoWj+vXrS+/eveWRRx7xe1fIRwV+bjzMzj77bDn++OPjj4cPHy6zZs1S1d/nnXeefPnll1KlShX1XEFBgZqyoaSkRMrKyqRSpUpZ2V5UVKxY0e9doCTTp0+XLVu2yO9+9zu/dyUQ8vPzVU12LsG46Hv27Il/VmYLrpmLLrpIvvvuOznssMOyum0KBtYcZdEZZ5whd955p3z//ffy0ksvpW1z9P7778upp54qNWvWlGrVqslRRx0lI0aMSHgNPjSw7JFHHqk+9A499FA5//zz5dtvv1XPo7YH68UvoMcff1wOP/xwqVy5sqoNgq+++kouvPBCqVWrlloehTlzVTJChPiAgNNPPz0eKjTXhrzzzjvym9/8RqpWrarCF7169ZIvvvhCO/w4b948ueGGG6ROnTrqWK+++mrZt2+fbNu2TQYMGCAHH3ywmm699Vb1QWm2a9cu+cMf/iCNGzdWx4VzhGNNft3evXvlpptuUtvAPqJw+u9//7vcPuF9ufbaa9V68GFcu3Ztdfw6tWbmNkd4PbYFqKGInTe8Vy+88IL6+5NPPkn5Cx01HP/5z38st4OwLGogsS0cM9rTnHnmmbJs2bL4a/75z3+q/W7SpIl6Dc4Pjj+5dhL7jGtr7dq1qtCOvxs2bChPPfWUev7zzz9X1yze26ZNm8rkyZNTvodz585V7xvOV1FRkXrffvzxxwOeM7wvI0eOlCOOOCK+n3ifMT/TeyEV1HbgPOG6Nx9z7PjM4e9Mr6lU7rvvPlUAefLJJzO6P2LvA973Pn36qL9x/fzxj3+U0tJS0YV76cQTT1T3Mr7QX3zxxQO2OVq9erVccMEFqrYEyzVq1EiFILdv3x4/Rzgnf/nLX+LnCvsbg+sYPwTxvmO/u3btKh9//HG5fUNteefOndV9hW3gXMXuBfP9FWs/9e6776rPI7x+4sSJ6jm8Htcjrnm8N0cffbSqmU8WWweOM7YO1NjHjvv1119Xj3G8HTp0SHkvduvWTf1bXFysff4pXFhzlGWXXHKJ+mB/77335Morr0z5Gnx44uZu27atCs/hg+Cbb76Rjz76KP4afGjiNTNnzlQfZjfeeKP64sQXyYoVKxK+EPChgoLUVVddpdaFwhC2ccopp6gvQzRAxIf3q6++qj6c//73v8tvf/tbVf2OggvaSmGfW7VqpdYX+xdhw0svvVS1p3rwwQdVKAkfVvgiwweOTgNlhBzxwYxCBD5Un3nmGfUlOH/+fPXljgIDwiAPP/ywtG7dWn3xAr6sUMj58MMP5fLLL5f27durD9RbbrlFfck89thj8W0grILC6P/93//JySefrGrw8CWVbPHixWq7OJ/4AMeHNo4HDYlRoDzooIO03mN8sWG5wYMHq/OIAivg/WzevLkMGTJENRA+9thjE5bDPGwL74mVa665Rl577TXVGBhfDj/88IP6UkRN5HHHHadeM3XqVPVeYPsosCxatEh9YaNAiOfMcB3hyw3v9UMPPaT2AevG9XD77bdL//791f5PmDBBnftOnTqpYzDD6/GeofCH0CKOHQXN2JdxKqi9xPuHfcd1iWsKhTG8b19//XU8jKNzL1jBexk7JzEoxK1fv17dJ7h+zTK5ppIhNI5rFV/ksfs6k/sD7wNeh4bAKIx98MEH8uijj6r7GO/jgeCc4IcO9hvbfP7551UhBl/+xxxzTMpl8CME20RhNHYf4jhR44YfJzVq1FDHgPsHhS68TxD7bMF7g4IfCkYo1KL2FMePa3jOnDnxRs1YZ+zHFWrQcW0999xz6r1MBddQv3791HuFc4kCKuDc4VjwHqGm/a233lI/ZnAt4Z5KPh+437GO3//+9+qcnnvuueo6xmcZloMxY8aoWiJsEwXbGBw7jhPXGX5YUAQZ5KoXXngBPzGNxYsXW76mRo0axrHHHht/PHLkSLVMzGOPPaYeb9myxXIdzz//vHrN2LFjyz1XVlam/l2zZo16TVFRkbF58+aE13Tt2tVo06aNsWfPnoTlTj75ZKNFixbxeVOnTlXr+PDDDxOW/+mnn4yaNWsaV155ZcL8jRs3quNLnm91nnr06BHfX+jUqZORl5dnXHPNNfF5JSUlRqNGjYzOnTvH573xxhtq+fvuuy9hvRdeeKFa/ptvvlGPP/30U/W6a6+9NuF1//d//6fm49zH7N69u9x+LliwQL3uxRdfjM/DuUg+J5deeqnRtGnT+GO8d8nrj+nXr5/RoEEDo7S0ND5v2bJl6vU4L+ng3A4ZMiTta1Idx5gxY9R5+f777xP2GdscPXp0fN6PP/5oVKlSRb12ypQp8flfffVVueOJvYcdOnQw9u3bF5//0EMPqfnFxcXxeXjvzO/fX//6VyM/P9/45z//mbCfEyZMUMt+9NFH2vdCKvv371fH8Ic//KHcczh/qT76dK8pwOti7wO2gWOZNGmSrfsj9j7cc889Ca/FZwTO7YHgusPyc+fOjc/D/V65cuWE40++bj/55BP1GPd4OlWrVlX7mKxPnz5GpUqVjG+//TY+b/369Ub16tWN0047LT7v+uuvV+cP24v54YcfjFq1aqnt43Mq+VhmzJihdV3j8+Owww5LeT7mz58fn/fuu++qebi2zffAxIkTU36+Qffu3Y1WrVqlPTcUXgyr+QDVz+my1vArPFali19FqaB255BDDok39jZL/rWOavNYmAe2bt2qak/wiwn78d///ldNqIXAL0lUtacL7QB+eePXJX7hxZbHhLAQfjHi17cO/NI17y+WxXcP5sdgnageR/w/BrVJmI+aLTOERLA8whmx10Hy6xCaSmZu14DGvDgfCPng/TCHrZxCDQxqL8znCDU22D7eq3SwLwsXLlTLWzEfB0IieF9QY4bzkiqEgJoB8/rxSx2/7s1tdTAPz5nfgxjUKJjbXKGmA7/sY+c+FdRgobaoZcuWCdcPwiYQOzc690IquMZxvAjJ6tK9pmIwD7VmaPSNmknU2Di5P1AraIZamVTnOxXUIuL1Mbjf8Z6lWx61I4DaMdRqZQI1Xaj9Rk2zuU0OQvuosUGN4I4dO9S8GTNmqBpH1MTFoPYatZKpoGYSn0PprmuE/XA+EarDMcbCgObzgW3GxGqxcH2hRjp5fqrzhGsH26BoYuHIB+h7Be0PrFx88cUq5IUvrXr16qkwD0Je5i8HtCvCh59OQ+7kMAiqnPHBjvZP+BA1T2gDAps3b067ThSgYh82yevAh+aBlo8xf1CZP7DR5iN5vrkdC8I26CIh+TzGQn54PvYvqsvNYUaIVdWboU3OXXfdFW9vgsInjgdfcskfvk6gjRC+RGJ97+B9/dvf/qYyZNJdF4DQF8Km2EeEOhDKSv5gRxsihFTwBRRrv4IvEUg+DrS7MBecY+caYcXkQnbyexDTokWLhMfYJo4vXVstXD8IyyRfO2g/B7HrR+deSEenrVCm11QM2vSg/RJCligEJR9fJvdHqvcBX846bbdS3Uc6y+Nz4eabb1YhLlzrKJDgeHSudTR0R4Eq1X2E84X3Z926dfHzhh8ZyVLNi+1XKghxoS0QCu4oNON8xdqeJe9zJp8rkOo84dph/3PRxTZHWYZ2H7iRrT4YYr+Q0MgVvy7/8Y9/qF9er7zyivqgxQerOS1ZR3KmR+yLBQ0+U/1Cg3T7Z14H2iSgrUIy3ew7q2NJNT+TLzo7UAuH9lmoVcKvTnxw4sMRX8iZ1FocCI4Nv66R/v/000+rD33UBKFtxIGgNgc1BNOmTVPXAtpioT0LGpmi7RB+0aPwhZqT2267TdXM4MsENYEoMCUfRybn3833APuBRrFjx45N+XzsS8zuvYCCId473cKFHSi0ffrpp/KnP/1JvS/Ypt37I9N72q33C+2acF2gZg7nE7VmaIeD9n8oIPshVWYafgyisTeuZ1wzuD6QcYvaPrQF8+K6xrWDQiNFEwtHWRZrBGpVKIlBbQc+DDDhwwCNPdFAFl8S+PWEmhCEVxD+yTSNPFYNjuViWRlWrH45xWpikDlyoHV4AdlTaLSKsKD5lz4y8GLPx/7FB2espi3G3CdRDBo6IzSCL4wYNGRHzVGmDvSLE6E1bAeNShGuwa/gA10TMaiVQYNSTKiBQKPj+++/XxWO0KgZDZqRXRRrvB4L83gFtSRocGuuGd2wYYP07NnTchlcP+hoD9f3gc7Vge6FVFD4wDbWrFlT7jmr7eleU+YfEKjJQwPks846SyVHxJbz+/7IBAqpmNCoHI3YUehDw2VklFmdL1yvSFBIdR/hfOE9ixVwcd5QW50s1TwruE/QcBzZtOZaId3wvR24dtq1a+fZ+inYGFbLIrTzuffee1W1sVW8HfCrP1ksXh9Lc0bbFMTD8as101+L+MDGBzoyS/AllqrKPAa1DpBcQMAXObJU8EWFAlq6dXgBX7yoJUk+fvyKxIc5CgoQ+xcZd2bo2iDVr8rkc4eQSSbp1DGxzDarghWyrzAhpIH2Y6idOlBtG/YjOXyA9xKhoNh1EftlbD4O/I12MV5BhqH5GkBWEfrTip37VFDTgtos1J6lCm+irZTuvWAFtX9LliwpN9/qmta9pszwHqL2AtmCyIaKdZfg9/2hA22C8D6ZoZCEgo353OJ8JZ8rXGfdu3dXNU7m8OmmTZtUlw/IyMPxx87FggULVC1bDN7XTIZ0SXVd415ATa8XsG78oEJbPYom1hx5BLUB+AWFDx98YKBghF/v+BWFXz/pOmNDyjJCCUg3x+tRO4DwC6q58aEDqBVAmwe0GUCqNkIt+ELBL1/UKKD9SjpoW4B14cMQ6bKoTcJ+4kMMoT/8qo99EeGDCaEbfGCgLU6srxF8CaJrAtRc4MsdvybR3gXhD/z6TFVwcwu+iFBbgRoEfDjjFx7CAviwRlgs9ssd+4/2IDh/2H982OEXfqpfrUgZR80ewmlo0IlzgfOJdHg7oQGsAyEgtKNByAVdEWCKwXuI0CbohNRQo4FrACnbOF607cH+oQuCWG0Xwg44dqwXhQ98QaHw5WV4CSnhqNWJpUTjXOPaQsq1FVw3aDuERsj49Y/rBQUT3DOYH+vnRudesIJ7AO8natJibZkA6e2AEBK+uHF94/rVvaaSnXTSSeo1KFzhvUE3BDjvft4fOvCZhAbl6BML5wefVThfOB/mxACcL1xnqLVDQRw/7tCQGTVLsT6o8JmDwj1+cKFghRq1GKT5o8E6wr0IXcdS+VEDhEKSTrseFMQQRsN7hPR81E6iYI3PoVQ/8JzC8aIgdqDPUQoxv9PlwiaW3hybkOpav35948wzzzSeeOIJY8eOHeWWSU7lnzlzptG7d2+V7o3l8S/Sv7/++utyqa2333670bx5c6NixYpqO0g7jqXWxlL5H3744ZT7itcNGDBALYflGzZsaJxzzjnGa6+9lvC6Z599VqXLVqhQoVzaK/5GOi3SkwsLC43DDz/cuOyyy4wlS5bY6vIgdi6SU7eRSoyUYjOkS990003q/GD/0QUBjtXcNQD8/PPPxg033GDUrl1brePcc8811q1bVy41HWnsAwcONA455BCjWrVq6riQwo7UYHMqs04qPyCVGKnYeA9TpfVv2LBBndMjjzzS0LF3717jlltuMdq1a6fSpXEs+Pvpp59OeN3KlSuNbt26qWPAsSBtfPny5eW6Ckh1TgEp98ccc0y5+Ti+Xr16lXsP58yZY1x11VXGwQcfrLbZv39/laqdvE5zKj8g/f/BBx9U20LaOZbH+Ro1apSxffv2jO4Fq/OF47/33nsT5qNrCKSX16lTR6WYm+893WvKnMofg64LCgoKjIsvvjjeTYPO/WH1PiR/LlhJfl+sznnydfvdd98ZgwYNUvuEfUNq/emnn2588MEHCevBPYDUfKTBY3nzvYAuKHB8eN8POuggtbw5hT4Gafy/+c1v1PuMbjnQtcS4cePU+tC9wYGOBd58802jbdu2al+bNWumrp1YlybJ3QGkWkeq98zqMxLv4amnnppyPyga8vA/vwtoRFGEsCjaDyFDDpmDuQY9ZA8cOFDVXJmHygkShLERekG7KKeNnsldqI1DTRNqgYL03mzcuFHVjk2ZMoU1RxHGNkdEPhYuEEpC6IW8gd6N8eWLLzryT/LQNehDDCE8hOSCVDCKtUdEcwMWjKKNbY6IfGjrgeFIkGGGTvR0hlkhe9AuS7fPLfIOGscjCQR9IKFt45///GfVIDyINaYPPPCA37tAAcDCEVGWoZFxLGXaPEgpUVihsTq6ykBmIxpgo5E6CkgY048oiNjmiIiIiFwxfvx4NcW6eMBgwWhXma5rDwwnhFpELIMe95Edna6ftGxgmyMiIiJyRaNGjVRocunSpaqfMXT9gvZbGC4oFdSio7sVjKeJsR/R1AAThknyE2uOiIiIyDO1atVSQx2ZBxSPwfiJ6KNv+vTpCX2HoY869NTuF7Y5+t84SBjbCl3/c6BBIiLyAuoi0JkrOtNET+RewbBH6JzVLUaKQXjRITCmdJCNi5AZCj9olJ8KOttFZ8Zm6JwVnan6iYUjEVUwSh6tmYiIyAvr1q3zbGBfFIyaN60mGzdnPuxRuqzPnTt3JswbOXKk3H333SlfjzEeURjCvmBZDJSNEQOs+pWqV69ewjw8xnw/sXAkEh8s8lTpKQWS2SCuFGxbB3VMeFzr+YUZLaPzetjx2q9DSxRd+G1G+2h3m37Ilf2k4Fwb5vlzhj2XsMxvj2yT0brc3C83ls903SWyX+bJ2wkDG7sNNUYoGH2/tJkUVXdeO7XjpzJp2uFfqkAXGy8P0tUaYZBvjKWHIZtiA3rPmTPHsoAURCwcmUadRsGoII+FozCpUClxDDud99e8jO71UKHqrx8Udq4hO9v0Q67sJwXn2jDPL6qe2OGj1TXk5nXmdF3pls943f9r4ZuN5hvVquepyaky+WUdKBiZC0fpYBy8I444Ij42H3rRx+DX6BE9Wf369VXfV2Z4jPl+YuGIQueHq38dSbv2xPlZ2WaNnqtz+njSbcOP80nuMb9/ZrrvpdP337xMj4nttJYp6L3l1wflv09tbz/5XFgdj+4xB/l+KDXKpNRwZz1utOvFgMSpIPyGwcAxnEwMBjS2aqOULSwcERERkSuGDx+u+jRq0qSJanw+efJkmT17trz77rvq+QEDBkjDhg1lzJgx6vGNN94onTt3lkcffVR69eqlhvpBFwDoMNRPLBwRERGFTJkYanJjPZnAcD0oAG3YsEFq1Kghbdu2VQWjM888Uz2/du3ahEy9k08+WRWg7rjjDhkxYoTqBBKZaq1btxY/sXBERERErvjzn/+c9nnUIiW76KKL1BQkLBxRoOm2Ecj0NW4sr9suQed1dvbZar1unzOrdQe5vUXU6V4bOsvrcvM6sWrDZ+fa3v52i19f3zP716x5n0v37RF5vjgr2y1T/7mznihi4YiIiChkSg1DTW6sJ4o4thoRERGRCWuOKBTshLjsMK9bN1zgZpgvGyECO6ELChan4VunoTivQrHJy5vvB6tQXElxHdMj6y43dI7Taci6xNgvYW+QHRYsHBEREYUMCjWlLBzZxrAaERERkQlrjijQnIaukqvBzT3v6mTEuB1SstNbccL+WITSvAxdOA1FUm6zCusm3z9OQ3l2rmHdkJkOO71lBzl7k2E1Z1hzRERERGTCmiMiIqKQYSq/M3mGEdEjN9mxY4fq5ryL9OZI4wGnU43tZueMbiyTKTsZMW7ul24YLWhhBHJvsNko8ureTu4E8rPnR8j27du1R7i3+3321Zf1pHp158Ghn34qk5atNnm6z0HEsBoRERFRUApHc+fOlXPPPVcaNGggeXl5arC5mP3798ttt90mbdq0kapVq6rXYDC79evXJ6xj69at0r9/f1WirVmzplx++eWyc+dOH46GiIgoGJDG79YURb6G1d555x356KOPpEOHDnL++efLtGnTpE+fPuo5VOFdeOGFcuWVV0q7du3kxx9/lBtvvFFKS0tlyZIl8XWcffbZavTfiRMnqgLVwIED5YQTTlCj/OpiWC33OQ2RWXGjet1pdX02QmZuhx8pPHQ6WkzHj/B1UMOK6ARythRnJaz22cq6roXV2h69OXJhNV8bZKNggykVvLnvv/9+wrw//elPcuKJJ8ratWulSZMm8uWXX8qMGTNk8eLFcvzxx6vXPPnkk9KzZ0955JFHVG0TERERUWjbHKHkivAbwmewYMEC9XesYATdunWT/Px8WbhwoY97SkRE5J8yF6coyplU/j179qg2SP369YtX7W3cuFHq1q2b8LqCggKpVauWes7K3r171WSuhqTcYFXdHrRQmpP1eTm2mZ0MP79DESS+hpXMHS3+cHUdrXW5OQagnRCbH9ds0MLPZZInpZLnynqiKCdqjtCW6He/+52gedT48eMdr2/MmDEqbBebGjdu7Mp+EhERUe7Lz5WC0ffff6/aIJkbhNWvX182b96c8PqSkhKVwYbnrAwfPlyF6GLTunXrPD0GIiKibCoz3JuiqCAXCkarV6+WDz/8UGrXrp3wfKdOnWTbtm2ydOlSlfEGs2bNkrKyMunYsaPleitXrqwmIiIiokAVjtAf0TfffBN/vGbNGvn0009Vm6FDDz1UpfIvW7ZMpk+frlL4Y+2I8HylSpWkVatWctZZZ6l0/wkTJqjC1HXXXSd9+/ZlplpIpGt/Y6ddhe4gmn62K/CjvUIQ2kiQe3QHYrZaxuo61+093Xw/+dFmSPezIUjdbLit1KU2R6URbXPka+EI/RWdfvrp8cc333yz+vfSSy+Vu+++W9588031uH379gnLoRapS5cu6u+XX35ZFYi6du2qstQuuOACGTduXFaPg4iIKEhYOMrhwhEKOOn6oNTpnxK1SJl0+EhERESUs22OiNJxWo1tFUoLclq7096KczE8QM7ez4LeW359ojjz8LPVfPO1mO569Pt60j3OXOkyQFeZkacmN9YTRSwcERERhQzDaiFP5SciIiKKzMCzQcGBZ3NHplXfXvY27fc2sxUKY8gtenSyvZxmvumGiZ1ef+ZtmHv71l2fboaezrqyOfDsrBWNpZoLA8/u/KlMzmi9LnIDz7LmiIiIiMiEbY6IiIhCxnCpQbYR0QbZDKsxrBaKTiDtDDxrp6M8q33xklXoQbejPqfnjHKbm9dzkK+fXOjQMZthtfc+bypVXQir7fqpTLq3+Z5hNSIiIqIoY1iNiIgoZEqNfDU5X49EEgtHFGh2Oqrzcpu6IQo3Zdq5Xrrxoyh63LxP7HQCma2OT90MpbkZojMr3bdH5HlTT5weKpM8KXMhOFQm0SwdMaxGREREZMKaIyIiopBhD9nOsHBEOctO5llQM7d098XO2Go650Y3241yg1eZnNnafoKEKFTiNZ+Ne9jNMdeQrZZ7bY4MiSKG1YiIiIhMWHNEREQUMr80yHYeEiuLaFiNNUdEREREJqw5opylOwimzvJOt+m0zY5u9wG1e3rTFiRdu4ogtc2i4NBNq/eqh+502/Gq+wI7zO0ES3ftFblQsgJp/KVM5beNhSMiIqKQYYNsZxhWIyIiIjJhzREFmm41utVr3EiFt1reHOLSrdK3s89upuJb9XCcLlzHUFruyfSaCUM3GXa27zQ0r8P8OZPNVH6E1dhDtn0sHBEREYVMqZGnJjfWE0UMqxERERGZsOaIIhU6cDo4ptPls5FFo7sNq2NhD9nhkjBAbJoxTzN9n3WzGtOFqzK9h3W36Uev4EFT6lK2WinDakRERBQGZUa+mpyvx5AoYliNiIiIyIQ1RxRoTjtqzBY/tqmb+ZZpx49hCy9EnTl8+sPVdVKH25BJVVzHtaxQO6+zI9N1exkyDlqIjmE1Z1hzRERERGTCmiMiIqKQKXMpDb9MoomFI8opTqv4rTqBtNOhopvZOenohL/c3iaFh+V1KnohJqfhs3TXZkHvLSnvR6fXrFch4+RjMe+/ORPQ6phL9+0ReT5NymAgO4HMlyiK5lETERERWWDNERERUci4N/BsvkRRnmFEtBMDkx07dkiNGjWki/SWgryKfu8OUcYYVqNssxMu082Qc3t/3NqGUxhbbbYUy/bt26WoqMjT77NxS0+SKtWc13/8vLNEbujwsaf7HETRLBISERERWWBYjYiIKGQYVnMmmkdNRERErhszZoyccMIJUr16dalbt6706dNHVq1alXaZSZMmSV5eXsJUWFgofmLNEYWOVbp+EHrBtbO8zjJ2ujLQbe/htPsEyr17INNe1cuZmHp28r6Ye+x2Sud6tNM1R65yr4fs/IxeP2fOHBkyZIgqIJWUlMiIESOke/fusnLlSqlatarlcmjPZC5EoYDkJxaOiIiIQqbMyFOTG+vJxIwZM8rVCqEGaenSpXLaaadZLofCUP369SUoGFYjIiIiT2zfvl39W6tWrbSv27lzpzRt2lQaN24svXv3li+++EL8xFR+pvIHjpfV247DBS5uM0hyYR/JH2G+NrJ9bNlM5X9gcWcpdCGVf8/OEhl2whxZt25dwj5XrlxZTemUlZXJeeedJ9u2bZN58+ZZvm7BggWyevVqadu2rTo3jzzyiMydO1cVkBo1aiR+YM0RERFRyJQZ+a5NgBodFLpiExpeHwjaHq1YsUKmTJmS9nWdOnWSAQMGSPv27aVz587y+uuvS506dWTiRIvGa1nANkdERESU1roUNUfpXHfddTJ9+nRVA5Rp7U/FihXl2GOPlW+++Ub8wsIRBY65eju5R12rzJt31y+P/92jQTutdTsNselmcWVaXZ8uo8ZO5pkfoUQKDqfZhk6zKp1maNrZFzel23872Z/ZUip5anJjPYCCkU4oEC11rr/+epk2bZrMnj1bmjdvLpkqLS2Vzz//XHr27Cl+YeGIiIgoZMwhMafryQRCaZMnT5bi4mLV19HGjRvVfITiqlSpov5GCK1hw4bx0Nw999wjJ510khxxxBGqfdLDDz8s33//vVxxxRXiFxaOiIiIyBXjx49X/3bp0iVh/gsvvCCXXXaZ+nvt2rWSn/9roevHH3+UK6+8UhWkDj74YOnQoYPMnz9fjj76aPELs9WYrRZouiEmq6pv3bCcU2527qh7zLrbzzREYWf7FA1hyyT1itU9V7pvj3z2/IisZKvdtbCbFFZz/n22Z+d+uafjBxx4loiIiCjKGFYjIiIKGb/aHIUFw2oMq5FP47H5HS4IUkYQRZMfYxW6uXyQO4EcvuAs18JqYzrNYFiNiIiIKMoYViMiIgoZQ/KkzIV+jgwX1pGLWDiKkKhkIdnJ3JJi8wN3w2pudjyZK6E0v0OGlH1evefp1mt1DxX03qIVJg/ztVlq5KvJjfVEUTSPmoiIiMgCa46IiIhCpszIU5Mb64kiXwtHGJAO3YQvXbpUNmzYoMZi6dOnT/x5JNKNHDlSnn32WdWl+CmnnKJ632zR4tfMoq1bt6pxXN566y3V4+YFF1wgTzzxhFSrVs2nowquMFch58o5sNq+nf1K7uAy02w73U4gM13+QM9Rdnk1/pcbYfpMO3XVXpfmYO5hDv+WSr6a3FhPFPl61Lt27ZJ27drJU089lfL5hx56SMaNGycTJkyQhQsXStWqVaVHjx6yZ8+e+Gv69+8vX3zxhbz//vvxEYCvuuqqLB4FERERhYmvNUdnn322mlJBrdHjjz8ud9xxh/Tu3VvNe/HFF6VevXryxhtvSN++feXLL7+UGTNmyOLFi+X4449Xr3nyySfVSL6PPPKINGjQIKvHQ0REFAQMqzkT2PqyNWvWqEHounXrFp+Hjq06duwoCxYsUI/xb82aNeMFI8DrEV5DTZOVvXv3qo6yzBMRERFRoBtko2AEqCkyw+PYc/i3bt26Cc8XFBRIrVq14q9JZcyYMTJq1ChP9pvcZaddg5s95aZrf+NVGwXdbdrpyVun+4B03R+ErV1G9Hp8X53xvWW1rjB0/5DpdnKpO5QyyVeTG+uJokge9fDhw1VX6LFp3bp1fu8SERGRa0qNPNemKAps4ah+/frq302bNiXMx+PYc/h38+bNCc+XlJSoDLbYa1KpXLmyGiPGPBEREREFOqzWvHlzVcCZOXOmtG/fXs1D2yC0JRo8eLB63KlTJ5Xij64AOnTooObNmjVLysrKVNskyn1Bq153M/XY6b7Y4ThFmnKCTshV9311eyDmbMvW4LRBCz+zQXYOF4527twp33zzTUIj7E8//VS1GWrSpIkMHTpU7rvvPtWvEQpLd955p8pAi/WF1KpVKznrrLPkyiuvVOn++/fvl+uuu05lsjFTjYiIosow8qXMhaE/jIgOH+Jr4WjJkiVy+umnxx/ffPPN6t9LL71UJk2aJLfeeqvqCwn9FqGG6NRTT1Wp+4WFhfFlXn75ZVUg6tq1a7wTSPSNRERERJRzhaMuXbqo/oys5OXlyT333KMmK6hlmjx5skd7SEFmJ8PMipvLu02nul63Sj8I1f2UXUEL9+SaXA0/l0qemtxYTxRFs76MiIiIKNcaZBMREZE9ZYY7janLrIM7ocbCEQVauk7XdAZxtdOJo52MFC9DF047vnRz36w7F6QgMb9PtXvOD0z4zWmnqsnLF/TektH16GUnjkG7N8pcapBdFtEG2dE8aiIiIiILrDkiIiIKmTLJU5Mb64kiFo4oZ+mEAZx2+qa7PjthrXfXL4//ffyowY7WZfWadK8zhwFKiutobT8I4QI6MPP7aR5PLR2nISav7se0y0+0sYyDz4BcujfcGvqjNKKdQDKsRkRERGTCmiMiIqKQYYNsZ/KMdL0wRgTGbKtRo4Z0kd5SkFcxZ8bycXObdri5TXZUR35mQZlDjOnCjNm6Tu2EPHkPBV+JsV9mS7Fs377dswHPY99nv5t5iVSqWsnx+vbt2ievdv2rp/scRNEsEhIRERFZYFiNiIgoZAyXstWMiGarseaIiIiIyIQ1Ry71empnsFIrfrQzspO66ia2kQgXp/dDptdD8usz3X75ezv1ve7VfZa87sT9Wa11z/MeIjMMHeLO8CF5EkUsHBEREYUMs9WcieZRExEREVlgzdEB6PZ6qjPYabqei3UGNE23TSu6y9upknezGp9pyOHlx/uZafq90/vMDr/vOQo3htWcYeGIiIgoZDi2mjMMqxERERGZsOYoA+aq94LeWxKe0+nFNl2VuJ1eeDMNRaXL6PGjF2AKD93Bbq2WSfd63YxRK3aubZ3Bes37onv8bobMk3v1DtrAp+QvhtWcYeGIiIgoZFg4coZhNSIiIiIT1hxlIKGqe2Lys+5VaeuGsuyE0jJdr9MQm27VP8N3uSfdtaFz3ei+5+nCV5muy4qdfbHDzZC1OZT/C4bV6FesOXKGNUdEREREJqw5IiIiChnWHDnDwpFNutkpuhkpOmNB2Qld6HJzbLh0YQhmyIVHptdvELZp59rSCeW5GQpP3qYf55lyn+FSH0WGRBPDakREREQmrDkiIiIKGYbVnGHhyKbkTiDN2Wu6GTV+j/mUjfCVnc7xKDd41Vmp7vJeXjNeZcXpsnOfMDRNZiwcOcOwGhEREZEJa46IiIhChjVHzrDmiIiIiMiENUc2Y/fpesp1M97vNPXXDjvrytYyFJ6BZ3W5uS6nA89mqy2S322eKPex5sgZFo6IiIhCxjDy1OTGeqKIYTUiIiIiE9YcmWwd1FEqVCr0PfSj2ztuttKadQaVrd0z89RjndAFwwjBZSet3M617XSAYzevJz9C1m6/jqIBvWO70UN2mQvryEUsHBEREYUM2xw5w7AaERERkQlrjkxqPb9QCvIqZm17mfaCm66HbZ3l3VZSXMf0KHUYIxlDadGQ7Ws78VqE1QcMv5mXyVbP07y2KVvYINsZ1hwRERGFNKzmxpSJMWPGyAknnCDVq1eXunXrSp8+fWTVqlUHXG7q1KnSsmVLKSwslDZt2sjbb78tfmLhiIiIiFwxZ84cGTJkiHz88cfy/vvvy/79+6V79+6ya9cuy2Xmz58v/fr1k8svv1w++eQTVaDCtGLFCvFLnmEYhkTcjh07pEaNGtJFetsOqzntBNE8kK1udb9T765fHv+7R4N2B9xHN/YlXcjNze1QdjnNNtNd3hwWs8pIS17e74xTK152HMl7KJhKjP0yW4pl+/btUlRU5On3WYe/3yQFVSs7Xl/Jrr2y9ILHbO/zli1bVA0SCk2nnXZaytdcfPHFqvA0ffr0+LyTTjpJ2rdvLxMmTBA/sOaIiIiIDljo2mGa9u7dq7UcClVQq1Yty9csWLBAunXrljCvR48ear5fWDgiIiIKGcOl9kbG/9ocNW7cWNVIxSa0LTqQsrIyGTp0qJxyyinSunVry9dt3LhR6tWrlzAPjzHfL8xW83EsqYSqf8n+2GRWoTSrfXRDpmNWUW7I2thiDjsbzdYYgH6EtRhKIzO0l3Gj0Yzxv3/XrVuXEFarXPnAITu0PUK7oXnz5kmuYeGIiIiI0ioqKsqozdF1112n2hDNnTtXGjVqlPa19evXl02bNiXMw2PM9wvDakRERCEdPsSNKRPI8ULBaNq0aTJr1ixp3rz5AZfp1KmTzJw5M2EeMt0w3y/MVnMpWy1boTiddeuGrlgNT25JN7ZZLmRk6d4bbm4/+ZxZZakyCy08spmt1nbqH6XCQc6z1Up375XPLnpEe5+vvfZamTx5shQXF8tRRx0Vn499qlKlivp7wIAB0rBhw3i7JaTyd+7cWR544AHp1auXTJkyRUaPHi3Lli1L21bJS6w5IiIiIleMHz9eFaS6dOkihx56aHx65ZVX4q9Zu3atbNiwIf745JNPVgWqZ555Rtq1ayevvfaavPHGG74VjIBtjoiIiEIGmWZ5Pgw8a2gEo2bPnl1u3kUXXaSmoGDNEREREZEJ2xzZbHPkR/udMLcZYruK3Ob3/ZCtgWOzsV672+Q9FHzZbHN0zCu3uNbm6IuLH/Z0n4OIYTUiIqKQMUwdODpdTxQFOqxWWloqd955p0oFRCv3ww8/XO69996EmCb+vuuuu1SDL7wGXZCvXu1NpgwRERGFX6Brjh588EHV8v0vf/mLHHPMMbJkyRIZOHCgqjK84YYb1GseeughGTdunHoNClEoTGFMlpUrV0phYaGt7VpVT5tTbwvk14FilYm2NpXR9v3gtEdhXVbn2auUcPI/rKUz8HE6dnqrtux5W3P/7Sxj5x62Wl63J3Ai1hyFuHCEvg969+6t+j2AZs2ayd/+9jdZtGhRvNbo8ccflzvuuEO9Dl588UU1JgvSAPv27evr/hMREUUpWy0sAh1WQ98H6DXz66+/Vo+XL1+uxmg5++yz1eM1a9aogenMo/miVqljx46+juZLREREuSvQNUfDhg1TLe9btmwpFSpUUG2Q7r//funfv796PjZib6aj+e7du1dNMdiGVpV6sbtV2FbhI6t12wlrFfTekvUQldNwA0NpuU/nfbYTSst0G+k4HUTW7fB3puFo3R62KZrQNNeVgWcNiaRA1xy9+uqr8vLLL6ueM9GNONoVPfLII+pfJ9BlOWqYYlPjxo1d22ciIqJgFI7yXJgkkgJdOLrllltU7RHaDrVp00YuueQSuemmm+LjscRG7M10NN/hw4erPhti07p16zw+EiIiIsoVgQ6r7d69W/LzE8tvCK+VlZWpv5GdhkIQ2iW1b98+HiJbuHChDB482HK9lStXVpMONzNNylWbm8J0Iqtd62gvYZtiXibzcJXb1fOs7o8GpxlimWZu2bk3nHI7c0znnCWci57J22A4mn7FbLUQF47OPfdc1caoSZMmKpX/k08+kbFjx8qgQYPU83l5eTJ06FC57777pEWLFvFU/gYNGkifPn383n0iIiLKQYEuHD355JOqsHPttdfK5s2bVaHn6quvVp0+xtx6662ya9cuueqqq2Tbtm1y6qmnyowZM2z3cURERJTr0FTIjeZChkQTx1ZLMbaaV50wJmeXmLOynIYe/B7/ys1zFqROMMkevofudaQa9fMXJtkcW+2wF0dIhYOcVxKU7t4j3w0YHbmx1QLdIJuIiIgo2wIdViMiIiIbGFdzhGG1FGE1v6u37WzTzfHIvBoXyu3tUHD4HbJ1e/teXZvpOnq02o7uvc37KfiyGlabdLvkuxBWK0NY7bL7GVYjIiIiijKG1YiIiEKGw4c4w8KRC2Mc6a7L6VhO6bg5HpnTKnkvj5PCKV1YzDw+oEx0tm6vrjndcJmX9zbvJzJjJ5DOMKxGREREZMKaIyIiorBBjY8btT4Ga46IiIiIIo81RweI3Sf0am2jvYPOYJK6y2cLU4LJqy4b7LTFsepJ3rJdks171Ykg3Ce8b8mMDbKdYeGIiIgobNgJpCMMqxERERGZsOboAEqK65gerc76oJNur1tneTd7F9YNMzIMkHuc9n6e7jWZXpuJ92nivWrnmtNZxu/rN3kg69o9eQ/Rr5jK7wwLR0RERGEU0ZCYGxhWIyIiIjJhzZHJjtcOlwpVKydkx+j2zutmiMppD9N2Qgd+9wTu5sC5lBvcvP7sXHNO7xO3w88660tYpjj5Wd439CuG1ZxhzRERERGRCWuOiIiIwoap/I6wcGRS8vYhYlQqTKietjPoo06nd17KVkaNmwP0MpSW29KFiHSuk+RrU+cadhqKtdNxpe7yTrdvtS/M6iR9CIe5ERLLkyhiWI2IiIjIhDVHREREYcOwmiMsHJnUen6hFORVdFyN7XcWmNPtmAU5242CI10njlav0w3LWl0PboZidcOCTjtbtdpGOrwfyBYWjrIbVvv3v/8tO3fuLDd///79MnfuXGd7Q0RERJQrhaMNGzbIiSeeKE2bNpWaNWvKgAEDEgpJW7duldNPP92r/SQiIiJd6J/IrSmCtMNqw4YNk/z8fFm4cKFs27ZNPUZh6L333pODDz5YvcYwjFB0AhmkMYqcjl9lZztOs9CcdghJuSFbnShmY2wzL8PHVvvsNKxIlA6+jt34SjZy+2vd+5qjDz74QMaNGyfHH3+8dOvWTT766CM59NBD5YwzzlC1RpCXF80SJhEREUWwcLR9+/Z4DRFUrlxZXn/9dWnWrJmqQdq8ebNX+0hERER2GmS7MUWQduHosMMOk88++yxhXkFBgUydOlU9d84553ixf0RERETBbHN09tlnyzPPPCMXXHBBygIS5iOTLZcVXfitSuW3kq6NQDbaBdgZqDJXethmu4rco9tOx6uuMfy+ZnSPS6fN1IHWkek2iVxrTG0Ev7kMKmgWL14stWvXTpiP9tHHHXecfPfdd94Vju6//37ZvXt36pUUFMjf//53+c9//pPxDhAREZG78oxfJjfWE3T/+te/pLS0tNz8vXv32i6XaBeOUAAqKipK+zzS/ImIiIi89uabb8b/fvfdd6VGjRrxxygszZw5U7WLtoM9ZAcwrd7O9t1cxkvmwULNXSb4PVgvecfqPXQ6cKyb7IS4nHZTUdB7S+KMifbXRRTFHrL79OkTz5S/9NJLE56rWLGiKhg9+uijttbNwhEREVHYRKDNUVlZmfq3efPmqs3RIYcc4tq6WTgiIiKinLVmzRrX18nCUQbShc6sqrvthNvsLONViMLtUJ7VvjFcED1Or1Pd7FGdnqidDihrZ3nd49e9t5nJRlELq5mhfREm9LkYq1GKef755yUrhSOkxy1atCjlTmDMNSIiIvJRhApHo0aNknvuuUeN4IGRO9wYrSPjwtFbb70l/fv3V4POInvNvBP4m4UjIiIiypYJEybIpEmT5JJLLnFtnXlGhqPFHnnkkdKzZ08ZPXq0HHTQQRIGO3bsUCmAXaR32k4graq63QgRvLt+efzv40cNTvmabFWVOw0FmiUvz6y0cNLN9spWiCjT6yxbHaz6vU3yV4mxX2ZLsRqOK13XOG58nzV+5F7Jr1LoeH1lP++RdX+809N9dgqdPyKadfjhh2d/+JAYdKh0ww03hKZgRERERLnriiuukMmTJ7u6zozDaj169JAlS5ao7rqJiIgogCKQyh+zZ88eNbzZBx98IG3btlV9HJmNHTtWPC8c9erVS2655RZZuXKltGnTptxOnHfeeRIFbmSEmUMMPRr8Or+2pM600a2GtzPmk1N2whUJoUmNDvAouHRDQub7xk72Z7bojIemmxEX1LHhKNyiNHzIZ599Ju3bt1d/r1ixIuE5u42zMy4cXXnllepftAxPhp1INb4JERERkRc+/PBD19eZcZsjpO5bTSwYERERBSiV340pQ3PnzpVzzz1XGjRooCpN3njjjbSvnz17tnpd8rRx40bxS4HTOF9hofPW8GFgp6M5nRCDbtV9pmGAVI8PFO6yk6Fnp0NIdmaXG9K9T25mm7nZEanuejMNhflxnbqdMUvkll27dkm7du1k0KBBcv7552svt2rVqoSMuLp162otd/rpp6cNn82aNUs8Lxyhdghp/OhXYNOmTfL111+rxtl33nmnGuTt8ssvz3gniIiIKBzOPvtsNWUKhaGaNWtmvFysvVHM/v375dNPP1Xtj5IHpPWscHT//ffLX/7yF3nooYfi7Y+gdevW8vjjj7NwRERE5DPUo7jSIFuyB4WcvXv3qvLE3XffLaeccorWco899ljK+VgHOqzOSpujF198UaXMoZfsChUqxOejCu2rr76ytRNEREQUXDt27EiYUIhxC4b8QDTq73//u5oaN24sXbp0kWXLljla7+9//3tb46rZ6iG7SpUqqhDUtGlTqV69uixfvlyF1ZDaf+KJJ9oupeVaD9nkP512VgW9t2TczstOWrab++z2NnW2T9Gge284XXe2rtlMt6nbtUhJcZ2M1hvEHrKbPnC/5LvQJrhszx75ftjt5eaPHDlS1cwcCNoCTZs2Tfr06ZPRdjt37ixNmjSRv/71r2IXlr3ttttk/fr13ofVjj76aPnnP/+pCkdmr732mhx77LEZ7wAREREFe+DZdevWJRToKleuLF5CZcu8efO0Xpvc6Bt1Phs2bFAdVqM9tB0ZF47uuusu1cAJw4ggff/1119XLcwRbps+fbqtnSAiIqLgKioqyurYamhQjXCbDtSUmeXn58tRRx2l+mPs3r17dgpHvXv3lrfeektttGrVqqqwdNxxx6l5Z555pq2dILJDK606Tc/bTtOyrUIU6cITOttMTtE2H4NOGMHNns91t0nBovOeJV+nTt/nhDCdjR7vy133McV6yzvd/8TzodeTu1f7EsSao0ygec0333wTf7xmzRpV2KlVq5YKlQ0fPlxVsKBSBZDM1bx5cznmmGNUF0HPPfecSr9/7733tLb3wgsviNsyLhz9+9//lt/85jfy/vvvl3vu448/lpNOOsmtfSMiIqIcGz5kyZIlqu+hmJtvvln9i6jTpEmTVMhr7dq18ef37dsnf/jDH1SBCYPaY3w0jJNmXoeOpUuXypdffqn+RkHLSVOfjAtHqKJCHBAlQLOPPvpIjbu2bds22ztDREREua1Lly6q3Y8VFJDMbr31VjXZtXnzZunbt6/qaTvWTxLKIihcTZkyRerU+bWBvWeFI9QMoYCEsUyQrWbuKlyn5XqmUJJEa/N33nlHdu/eLUcccYSqQjv++OPV83gD0Gr+2WefVScD/SKMHz9eWrSwqKKlnOZmdoqT9SZzs3dic6bML1ZnFLpwM6POjfWRv9wMs3qZ7Wa9vF6Y2uo4vept3e3thCmslm3XX3+9/PTTT/LFF19Iq1at1Dxk0KOm6oYbbpC//e1v3vdzhFggYoYoDKGfAxSSUGOENkg33XSTuOnHH39UhZ2KFSuqwhEO9tFHH5WDDz44/hp0Rjlu3DjVR8LChQtVO6gePXqouCUREVEk+Ti2WrbNmDFDnn766XjBKJZZ/9RTT6mygx0Z1xyhFTiqqVAgOuOMM+Szzz6TMWPGyHXXXSdue/DBB1VnUObGVmi0FYNaIzTkuuOOO1RDcUADr3r16qmB7lDNRkREROFVVlamKlGSYR6e86xwhAJQMoTQ+vXrp3qgPO200+KvQUMqt7z55puqFuiiiy6SOXPmSMOGDeXaa6+ND1uCFvAYtbdbt24JKX0dO3aUBQsWsHBE2tXbfle364Y+dDqxdLovFC5u3gO6maBeXXNOO0v1I2Rs3mbpvj0iz2um3+Vwg+xsQ0XNjTfeqMJnDRo0iDfJQTSra9eu3hWOMN4Jerk0N7CKPZ44caIaTgR/Yx4GpnXLd999p9oPoaX7iBEjZPHixSp+WKlSJRVLRMEIUFNkhsex51JBONDc9Tl6FCUiIqLc86c//UnOO+88adasmYo2xTqtxBhtL730kneFI9TQ+AHVYWh4PXr0aPUYaXkYZRfti+yOtAsIA44aNcrFPSUiIgoQI++XyY31BBwKRBiHDen/sTFe0f7IHFXypHCUPFRItqB3TDSqMsMBY2A6qF+/vvp306ZNCT1p4jFqu6ygA6pYvwuxmqNYaZOCJVvZZnb2RzdE5uY+u5l5Y+50zxyuY7gt9wXpmrOzbqv5ydtP6DjSYfam1f2gK3D3TQSy1WbNmqXaO6OPRfTejY6oY51RY/w69HWEyhT0zeh5thp8++23KnUOpTJMCHVhntuQqYahScy+/vrreGENjbNRQJo5c2ZCQQdZa506dbJcL8aEiXWFnu0u0YmIiMg5JGShDXKq73C0P7766qtl7NixttadceHo3XffVbU5ixYtUo2vMaEwghJaql6znUBjKpQIEVZDV+STJ09W7ZuGDBminkcbp6FDh8p9992nGm9//vnnMmDAANUgK9MRgImIiMIi1iDbjSmoli9fLmeddZbl8+iTEb1m25FnpOvGMgW0+0EG2QMPPJAwf9iwYWocFMT93ITBbBEGW716taopQjgslq1m7gQShSZ0Annqqaeq/g6OPPJI7W2gtgmlzC7SWwryyqcDUnhYZbsErkrcI8xWiwY777NVWClbYbVMw79e71umIXOdfSkx9stsKVYhH68iFrHvs8PuGi35hYWO11e2Z498d88IT/fZrsLCQtUOGZ1Dp4JKlTZt2sjPP//sfT9HGLfk1VdfLTd/0KBBqorLbeecc46arKD2CB1QYiIiIqJoaNiwYdrCEboYMrdH9jSshjFKMLpuMsyrW7eurZ0gIiIiF7kVUjMksHr27Cl33nlnyhExUFuEqFK6yhVXao5QM/PHP/5RhbSuuuoq1QfRySefHB90Fr1ZmzPAiIIoSKEknUyd5OfCdPwULFYZWm523Ji8Lp11W4X4dLeZ6fbsMo8194MEIHwdgWy1O+64Q15//XXVjAZZa0cddZSaj3R+DB2Cfhdvv/12bwtH6BfommuuUaU0DDiLMc7QFgjQABo9ZiNrjYiIiMhr6PB5/vz5MnjwYFUeiTWhRnMbtI1GASm5k2jXC0fmjSKLDBNGwQUUloiIiCggIlBzBOja5+2331YD1aMBNsoqLVq0SBig3o6MGmSjYGTGQhERERH5DYWhE044wbX1aafy5+fnq/TA5AJSsq1bt0quYSp/7nDai61X/OhF2Kv1Oh3ck4Ir3bWUaW/VdiT0aJ2lLgPM6zW3C4KS4jqOtpnpPmczlf/wEaOlggup/KV79si3o4OZyu+ljGqO0O4IJ52IiIgorDIqHPXt25fp+kRERBRq2oWjA4XTiLxQLq29pzfV/em2mbB9i+14ObilV4PtMpQWDbrp706vbR3J17z5fjDf21bsdHORcJ1L5ss73b5vItIg2ysZZ6sRERFRsLk1LlpeRL/6tQtHZWVl3u4JERERUQBkPLYaUTalq7bWqdLWrQbP1gCWOqG0bIW4OAhtNOi+t1ZZXV5mhXqVrWa1Ljvnws7ygRHRWh83ZDy2GhEREVGYseaIiIgobNggOzudQIYZO4EMLjudxmUrXBSksJTuvrDjx2jzKnSVzI/70Q9B7gSyxa2jpUJlFzqB3LtHVj8UvU4gGVYjIiIiMmFYjYiIKGwYVnOEhSMKtHSZMl51WqcbLtDJfNPdNztjXtmRjY7+KFi8GhstudNGq+ve72vLy7Ce38eWDvs5coZhNSIiIiIT1hwRERGFDcNqjrBwRKGgm4WV8VhMScvrjI3mtKM5O+sLcvU++cvLsdG82o7T8QiJhSOnGFYjIiIiMmHNERERUciwQbYzrDkiIiIiMmHNEeVUD9klxXUOmFbvaer7RPdSl4Paq3e6dlph7u04rJy+Z3685wnXYLH5mdWOBtH9QTJvg6jbY3/gsM2RIywcERERhQ0LR44wrEZERERkwpojCrTkausfrv41rGbFzsCrdrjZE3e2whV2eujWWZ6Cy6vrPFshJsddY0zMPCxnJd0xBi3kzAbZzrBwREREFDYMqznCsBoRERGRCWuOKKf4HQrzauBaO1Xy6bZjtU2nghY6oNScXk86y3iZqWXn2vLjegzy/cCwmjMsHBEREYUNw2qOMKxGREREZMKaI8pZQa3S9iqMobud5HCbznbsdFxJweVmx51OOzvVXbfO+rzsoNTO8oG+H1hz5AhrjoiIiIhMWHNEREQUMnn/m9xYTxTlGYYR0UqzX+3YsUNq1KghXaS3FORV9Ht3KA1zx3NeZcukq173fZypXKnSp1DSvf6txjZLFuixyTxQYuyX2VIs27dvl6KiIk+/z44ePFoqVC50vL7SvXtk5fgRnu5zEDGsRkRERGTCsBoREVHIsJ8jZ1g4okArl53Sc35GobeS4joZh6KylW2myyojR2e/kpen6LETinY1k3Ji5tvJVihZZztBzYo9IGarOcKwGhEREZEJa46IiIjCKKK1Pm5gzRERERGRCWuOKGdZtQVIbFeRnVRh3cFmndJpC5FT7SIop9oZpWsXZKedjs61mtAVQJr2S2Zubj9X7yc2yHaGhSMiIqKwYYNsRxhWIyIiItfMnTtXzj33XGnQoIHk5eXJG2+8ccBlZs+eLccdd5xUrlxZjjjiCJk0aZL4iTVHlLOcDlRp9u765fG/ezRo52ibudIrt5315mxac8QkdmGhF1bTeT/N4TqdbjXsSgjRycmeHEvYr3M/w2q7du2Sdu3ayaBBg+T8888/4OvXrFkjvXr1kmuuuUZefvllmTlzplxxxRVy6KGHSo8ePcQPLBwRERGFjY9htbPPPltNuiZMmCDNmzeXRx99VD1u1aqVzJs3Tx577DHfCkcMqxEREZFvFixYIN26dUuYh0IR5vuFNUdEBwilZUO2wlo6y6cLReZqiCFqnPbwrJcJqrd9XXayL90Mf2Xj3srlsNqOHTsS5qNtECY3bNy4UerVq5cwD4+xzZ9//lmqVKki2caaIyIiorCG1dyYRKRx48ZSo0aN+DRmzBgJM9YcERERUVrr1q2ToqKi+GO3ao2gfv36smnTpoR5eIzt+VFrBCwcUaA5rd62s4xuR3duSlcln+k2zRlFyZlLfgzuqYOD5XrHzc4Z3c5wND9np7PHTDNWdTNB7Rxn4K5ZlxtkFxUVJRSO3NSpUyd5++23E+a9//77ar5fciqs9sADD6g+E4YOHRqft2fPHhkyZIjUrl1bqlWrJhdccEG5EigRERFlx86dO+XTTz9VUyxVH3+vXbtWPR4+fLgMGDAg/nqk8H/33Xdy6623yldffSVPP/20vPrqq3LTTTf5dgw5UzhavHixTJw4Udq2bZswHyfvrbfekqlTp8qcOXNk/fr1Wv0qEBERhVWsQbYbU6aWLFkixx57rJrg5ptvVn/fdddd6vGGDRviBSVAGv8//vEPVVuE/pGQ0v/cc8/5lsYPeYZhGLlQCkXPmShN3nfffdK+fXt5/PHHZfv27VKnTh2ZPHmyXHjhheq1KHWijwSkAJ500kla60eLeDQw6yK9pSCvosdHQ17wOyzmlG41vp0wgNVz5vCbVejN7n5S8Dm9N9weT9DOeHC5psTYL7OlWH13eRWiin2ftRswWipUKnS8vtJ9e2T5iyM83ecgyomaI4TN0Htmcj8IS5culf379yfMb9mypTRp0iRt/wh79+5VF5B5IiIiIsqJBtlTpkyRZcuWqbBaqr4RKlWqJDVr1izXPwKes4IUxFGjRnmyv0RERH7LMww1ubGeKCoIeurgjTfeqOKQhYXOqwdj0BgMMdAY1ByhDwcKn1zJvPIjI4ahtPDSyUT0sqNDO9eMVSjNaYZcOk5DiU4z3MI6fEgYBDqshrDZ5s2bVXujgoICNaHR9bhx49TfqCHat2+fbNu2LWE5ZKuh3wQr6J8hlpboZXoiERER5Z5A1xx17dpVPv/884R5AwcOVO2KbrvtNlXbU7FiRTWCL1L4YdWqVaoVvJ/9IxAREYVp+JCoCXThqHr16tK6deuEeVWrVlV9GsXmX3755SpEVqtWLVUDdP3116uCkW6mGhEREVHOFI50PPbYY5Kfn69qjpCFhn4RkPJP0eV37N/tthw6x2Nnm3YGnqXc4Pd7lq2e7d3cvtP1+n3Oy2Gbo2gVjmbPnp3wGA21n3rqKTURERERw2qhbpBNRERElG05V3NE0WYVYjL//e765fG/e0xsZ7l8kEJMXg4Cmuk+By48QFnvPd7NwZt16d6DVhLu+waJ972fzPuP3qbl+eLsbJhhNUdYOCIiIgoZhtWcYViNiIiIyIQ1RxRoumEt8+t6NEg9mKVavmdwQky64Y5sbIehtPDSfW8T7pViZ723O+3V2rxMQe8tv75oovXyTkNpTu8Hq4FzzevCwLNZw7CaIywcERERhVBUQ2JuYFiNiIiIyIQ1RxRoTgdkTR7M0s1QkpuhAzM7nUCm42bHkZQbbHUcagpZ/XB1HUfb1A2F6YSQzfewl9mjjjuuTEhCWx2AbDXjl8mN9UQQa46IiIiITFhzREREFDJM5XeGhSMKNTvV8HbGOfMy8yvT9SUfsznEUVJcx5NQIAWLm+P72epsdKKz/dTdpt/Xps42ma2WmxhWIyIiIjJhzREREVHI5JX9Mrmxnihi4YgCzWl2itPsLt3xp7LVUV6m23AjCylIYQzK/r1lp4NSp8vYybDktZmEYTVHGFYjIiIiMmHNERERUcgwW80ZFo4op7y7frmjsZQyrXrXzs6xwU62mNPtZ5qtZ3c7lIOdQPoQpnaT1b5ZjXlm93U6HV/qLu8pdgLpCMNqRERERCasOSIiIgoZhtWcYc0RERERkUmeYUQ0oGiyY8cOqVGjhrQdNFoqVCp0tXdZq9e40RZEZ3k76a12Yu9uYkpu7su199B8zbt93Tvt7TnXziVZQw/Zs6VYtm/fLkVFRZ5+n3U8514pqFjoeH0l+/fIwul3errPQcSwGhERUcgwrOYMw2pEREREJqw5Mqn1/EIpyKvo2frTVYnrhLKylXpr3r5ulb55/+0MbmpmTonVHcCS/OV3+r/T7XsZPtYNkTFkRq5iKr8jLBwRERGFDMNqzjCsRkRERGTCmiOTrYM6qmw1s2xljViFsqz2RVe6fdbJjtENcSWGJZyFKALRuyxlJOE6KTfYrbP7xk5vz5lmj9q5N3Qzz8znpnZP7wY1puwL9PvEgWcdYc0RERERkQlrjoiIiEKGbY6cYeHIpWw1NzthdLN6VnfgVMv9TBNK86rTukBXVVNK5gzFX6zOaOBRpwP82slWS7dfbg7W6mb2aTq8b7Iv0Oe5zPhlcmM9EcSwGhEREZEJa46IiIjChg2yHWHhKMMxzKxeoxM6SLcOP6pnnWb0BLpKmbLKacjYTlhM9z7zavl0nN4b2erwlcIrz6X2QnkSTQyrEREREZmw5oiIiChsOHyIIywcmex47XCpULVyQkdtXmaeZRo60N2m7jKZZvTo0g0xupkRRNHrRFK3E0ad68xOiM3LsLjfIXfKfUzld4ZhNSIiIiIT1hwRERGFDbPVHGHNEREREZEJa45Mii78Nm0P2XZ61HXaxkB34FenrAbKdHsQWLafiDbLtj3FotXDtu56M+2awk77JS+vZd4n5FSeYajJjfVEEQtHREREYVP2v8mN9UQQw2pEREREJqw5yoBuuq9uL8A6yzgNa6VLCbYz2Gym27SzDEMKucdOj/HZGmDZ6TbNy7y7fnn87x4N2rnW+7ydfbPzeULRwbCaMywcERERhQ2z1RxhWI2IiIjIhDVHGchWVbXTEJObg4DaydDTzfzRXYZyTzbCpH6EYtOF0tzsfX772y1ShtbN88tn+BGZcPgQR1g4IiIiChkOH+IMw2pEREREJqw5spltpZ35lWaZTLeZrewUq3WlGxzUSkIYAOu2GNTXjJlrucfpILBub9OzTE6HdO9hq1BaSXEdT/aLQohhNUdYc0RERESueuqpp6RZs2ZSWFgoHTt2lEWLFlm+dtKkSZKXl5cwYTk/sXBEREQUMnll7k2ZeuWVV+Tmm2+WkSNHyrJly6Rdu3bSo0cP2bx5s+UyRUVFsmHDhvj0/fffi58YVktBpwO7dMvYqbrXWZfTavTkEJeZVTW+eb65Sv8XB+6g0k4nlgwX5Abz9btk5PiE544fNTirY6PZ6VAx3Wu8ugbtrJehNMq1sNrYsWPlyiuvlIEDB6rHEyZMkH/84x/y/PPPy7Bhw1Iug9qi+vXrS1Cw5oiIiIjS2rFjR8K0d+/elK/bt2+fLF26VLp16xafl5+frx4vWLDAcv07d+6Upk2bSuPGjaV3797yxRdfiJ8CXTgaM2aMnHDCCVK9enWpW7eu9OnTR1atWpXwmj179siQIUOkdu3aUq1aNbngggtk06ZNvu0zERFRYHrIdmMSUYWWGjVqxCd8P6fy3//+V0pLS6VevXoJ8/F448aNKZc56qijVK1ScXGxvPTSS1JWViYnn3yy/Pvf/xa/BDqsNmfOHFXwQQGppKRERowYId27d5eVK1dK1apV1WtuuukmVV03depU9YZdd911cv7558tHH33kW4jNHJYyZ2c5HUvJjoRtFjsbz41V+mR1PRwvgy2fs8PNzkK9yjyzCj/rbjM5zG1eh074MFsZdpSb3B5bbd26dapdUEzlypXFLZ06dVJTDApGrVq1kokTJ8q9994rfgh04WjGjBnlWrSjBglVdqeddpps375d/vznP8vkyZPljDPOUK954YUX1En9+OOP5aSTTvJpz4mIiMKjqKgooXBk5ZBDDpEKFSqUi+DgsW6boooVK8qxxx4r33zzjfgl0GG1ZCgMQa1atdS/KCTt378/IbbZsmVLadKkSdrYJmKlyfFTIiKi0DXIdmPKQKVKlaRDhw4yc+bM+DyEyfDYXDuUDsJyn3/+uRx66KHil5wpHOHkDh06VE455RRp3bq1mof4Jd6ImjVrasc2AbFSc+wUsVQiIiJyDmn8zz77rPzlL3+RL7/8UgYPHiy7du2KZ68NGDBAhg8fHn/9PffcI++995589913KvX/97//vUrlv+KKK3w7hkCH1czQ9mjFihUyb948x+vCm4I3LwY1R1YFJDupw1612UnXRsHOPuss4za2iwgPO++lVTud5N7Xrdrf2KHT5sdOlxNO9zHdNu1058H7iRKgwqfMpfVk6OKLL5YtW7bIXXfdpSoq2rdvr5rJxBppr127VmWwxfz4448q9R+vPfjgg1XN0/z58+Xoo48Wv+RE4QiNrKdPny5z586VRo0axecjfom0wW3btiXUHh0otomGZG42JiMiIgpzg2w739uYUpk9e3bC48cee0xNQRLosJphGOrkTps2TWbNmiXNmzdPeB6lSzTcMsc2keqPUqlubJOIiIgoZ2qOEEpDJhr6PkBfR7F2RGgnVKVKFfXv5ZdfrkJkaKSNlvTXX3+9Khi5lanmVW/VttKYkwZttQpr6Kb+6mzf7eP0KsQRFUEKS9rpodoyXV3z2tbpMkNXYo/vqwM18K6bXRlQRKk+itzoIVsiKdCFo/HjfxmSoEuXLgnzka5/2WWXqb9RFYfYJTp/RBYaxm95+umnfdlfIiKiqA8fEgYFQQ+rHQhG7sXov5iIiIiInMozdEogIYdsNYToukhvKcirKFEPnfi9fQoXnevJy97js7F93UxSN7eZbn28b4OpxNgvs6VY9dmn06Gik++zM9rcJgUVnCcelZTulVmfP+jpPgdRoGuOiIiIKPey1XJdoLPViIiIiLKNYTUfw2rZqAZ3OlBlus7t/NhnigadbC0/rk07eD2TH2G1rsfc4lpYbeYXD0curMaaIyIiIiITtjkiIiIKG6byO8LCUQYdEupWj9sZ20xneV26y+uE0syvSc7IkYniCYYeoiHddapzDQQpk9Pp+G+62yHSxsKRIwyrEREREZmw5oiIiChsypBy5dJ6IojZajaz1dzowC3TqvN02wxqR3u6+2yFIYVo0rme7IS1dMZpSxdOL+i9JeVr0q2PYwWSH9lq3Y682bVstQ++HstsNSIiIqIoY1iNiIgobNgg2xGG1UzVkG0HjZYKlQpdDZF5mXnmNCyVadW/nbCc7jK5mJ2Ti/scVHbGJvOyQ8lceD9zpRNM8imsdvhQ98Jq3z7OsBoRERFRlDGsRkREFDYMqznCmiMiIiIiE7Y5cmng2SC1P9Ht4ZopxuSEnWvezV7mnd5nXrbnc3t/dPbL788dClibo8NukIJ8F9ocle2VD74bF7k2RwyrERERhQ3Dao4wrEZERERkwpqjA9BNd/eq6j8dq+1YbbOkuE7SHG/SpXM9RZq8o/v+Z9ortp2wXrr5mV6nbndzkem9nW4Ziqgy1PgYLq0nelg4IiIiChuj7JfJjfVEEMNqRERERCbMVnMpW02H2yEmr8INXoYIc6HqPxf2MWjsDDCcK+fWj/3nNRhOWc1WazzYvWy1deMjl63GmiMiIiIiE7Y5IiIiChs2yHaEhSMPZKujOp3QhdW6dPfN7Sp9P7L6MhWkfckV6TpRzPQ6tcPLrEingyV7Nfg0M0EpLfZz5AjDakREREQmrDkiIiIKGxVVc6PmSCKJhSOTHa8dLhWqVnY85phu6CjT6nbdanQ71et+V8nrdrZJwaEb8jW/t7V7enOduT22m87yuqzORflOWVMvQ2QLw2qOMKxGREREZMKaIyIiorApQ8/WZS6tJ3pYODIpefsQMSoV2hpzzM5YSE6zxTINETgN67nBcpsT/d0vyly69+bd9cvjf/doIJ6Mh+bmtZEuXKZ7P+swh9KcrisI9zMFGMNqjjCsRkRERGTCmiMiIqKwYc2RI6w5IiIiIjJhzZFL3OwJ2856ddobZKtHXd0UbytsL5F7kq8tq3ZGBb23pGxnls6SkePjfx8/arCjfdPt/sK8jJuDMnt5bfO+oQQcPsQRFo6IiIhCxjDK1OTGeqKIYTUiIiIiE9YcmdR6fqEU5FX0JT3Waa/WTgf0dPM4vRxsl4JJt8uKH8R8nep1mdGjQbtf1yWZh4zd7DHezfvM6b4QHbAhtRshMYNhNSIiIgoDVahh4cguhtWIiIiITFhzZLJ1UEepUKnQ8SCwXoWF3A4dWK3L77AWwwi5J124KCFDrVhvea/uG17nFBkY9iPPhcbUBhtkExEREUUea46IiIjChm2OHMkzjIgeucmOHTukRo0a0kV6l8tWyxXb324R/7tGT70sIL/DChRe2QgtZytkne19SbcdM96zwaJzPZQY+2W2FMv27dulqKjI0++zMw7qKwV5lRyvr8TYJ7N2T/F0n4OIYTUiIiIiE4bViIiIwoZhNUdYOPK4utxyXCnN8Jdup3G1e2a/48VcC51Q9mT6vulmf9pZn1fj+bk9nqLuuG8UTIF7n9ABZB4LR3YxrEZERERkwpojIiKisFE1Pm70c2RIFLFwlMXq0aBlkfmdkWO17sBVT5MndK8tOyE2v8f301neTvh6ycjxKcecI0pmlBliuBBWMyJaOGJYjYiIiCiMhaOnnnpKmjVrJoWFhdKxY0dZtGiR37tERETkDwz74daUhe/kqVOnSsuWLdXr27RpI2+//bb4KRSFo1deeUVuvvlmGTlypCxbtkzatWsnPXr0kM2bN/u9a0RERJHySobfyfPnz5d+/frJ5ZdfLp988on06dNHTStWrBC/hKKHbJRKTzjhBPnTn/6kHpeVlUnjxo3l+uuvl2HDhvnaQ3a6tgfmXq1LiuukfJ3Va5Jf51UbCbfbHFn15M30/Wjzu4dq3a4EzN1xBK0NIQVfNnvI7pL3W1e+z0qwz8a0jPY50+/kiy++WHbt2iXTp0+PzzvppJOkffv2MmHCBPFDztcc7du3T5YuXSrdunWLz8vPz1ePFyxY4Ou+ERERRSmsts/GdzLmm18PqGny8zs857PV/vvf/0ppaanUq1cvYT4ef/XVVymX2bt3r5piUCKGEtnvSoeiZqX79iSUwBOe27X3gK+zek2516XZjpP9dLrectsxHY+X26Hcku7a1lnG6TWje2/lWVy/2dpPym3qOyZLGWBufZ+V/G+fUSNlVrlyZTW58Z28cePGlK/HfN8YOe4///mP6iN9/vz5CfNvueUW48QTT0y5zMiRI2P9qnPixIkTJ05ZndatW+fZd+LPP/9s1K9f39X9rVatWrl5+B516zu5YsWKxuTJkxPmPfXUU0bdunUNv+R8zdEhhxwiFSpUkE2bNiXMx+P69eunXGb48OGqsVjMtm3bpGnTprJ27VoVq40a/CJAPHjdunWRGnXZjOeA5yDqxw88B96eA9QY/fTTT9KgQQPxCrK91qxZo8Jbbu53Xl5ewrxUtUZ2v5MxP5PXZ0POF44qVaokHTp0kJkzZ6rW7bHGX3h83XXXpVzGqjoQBaOofiAAjj3Kxw88BzwHUT9+4Dnw7hxk4wc4CkiYcuU7uVOnTur5oUOHxue9//77ar5fcr5wBKgFuvTSS+X444+XE088UR5//HHV8n3gwIF+7xoREVGk3HyA7+QBAwZIw4YNZcyYMerxjTfeKJ07d5ZHH31UevXqJVOmTJElS5bIM88849sxhKJwhDTALVu2yF133aUacCH9b8aMGeUaeBEREZG/38lr165VGWwxJ598skyePFnuuOMOGTFihLRo0ULeeOMNad26tW/HEIrCEaC6zqrK7kAQYkNnVVYx1LCL+vEDzwHPQdSPH3gOeA6y8Z08e/bscvMuuugiNQVFKDqBJCIiInJLzncCSUREROQmFo6IiIiITFg4IiIiIjKJfOHoqaeekmbNmqk+ITBY3qJFiySskDaJwQCrV68udevWVX1QrFq1KuE1e/bskSFDhkjt2rWlWrVqcsEFF5TrnCssHnjgAdWxmblvjSgc/3/+8x/5/e9/r46xSpUq0qZNG5U2G4NmiMgyOfTQQ9XzGPNo9Wq9QVZzAYY2uPPOO6V58+bq+A4//HC59957E4Z0CNs5mDt3rpx77rmq80Fc88gEMtM53q1bt0r//v1V3z81a9ZUI6jv3LlTcv349+/fL7fddpu6D6pWrapeg1Tz9evXh+b4KXORLhy98sorqj8GZCYsW7ZM2rVrpwa727x5s4TRnDlz1Bf/xx9/rDrYwodC9+7dVf8TMTfddJO89dZbMnXqVPV6fECcf/75EjaLFy+WiRMnStu2bRPmh/34f/zxRznllFOkYsWK8s4778jKlStV3yIHH3xw/DUPPfSQjBs3To2GvXDhQvWFgfsCBccwePDBB2X8+PFqxPAvv/xSPcYxP/nkk6E9B7jH8fmGH4Op6BwvCgZffPGF+uzA6OkocFx11VWS68e/e/du9fmPAjP+ff3119WPxvPOOy/hdbl8/GSDEWEY52XIkCHxx6WlpUaDBg2MMWPGGFGwefNmNQbOnDlz1ONt27apMW6mTp0af82XX36pXrNgwQIjLH766SejRYsWxvvvv2907tzZuPHGGyNz/Lfddptx6qmnWj5fVlamxmV6+OGH4/NwXipXrmz87W9/M8KgV69exqBBgxLmnX/++Ub//v0jcQ5wPU+bNi3+WOd4V65cqZZbvHhx/DXvvPOOkZeXp8bSyuXjT2XRokXqdd9//33ojp/0RLbmCOPOLF26VFUfx6BTKjxesGCBRMH27dvVv7Vq1VL/4nygNsl8Tlq2bClNmjQJ1TlB7Rl6YTUfZ1SO/80331S91qI/EYRWjz32WHn22Wfjz2NMJnTaZj4HGO4AIeewnAN0OIehCr7++mv1ePny5TJv3jw5++yzI3MOzHSOF/8ilIRrJwavx2cmaprC+NmI8BuOOYrHTyHqBDJT//3vf1Xbg+RetPH4q6++krDDWDdoa4MQS6wXUnxAYlyc2AeC+ZzguTBAt/SoOkdYLVkUjv+7775TISWEk9ETLc7DDTfcoI4b3f3HjjPVfRGWczBs2DA1uCgKvhggE58D999/vwqbQBTOgZnO8eJfFKbNCgoK1A+rsJ0ThBLRBqlfv37xsdWidPwU8cJR1KH2ZMWKFeoXc1RglG2M4YM2A34NyhiEQjF+/Y4ePVo9Rs0RrgO0NUHhKApeffVVefnll9VwBcccc4x8+umn6ocCGuJG5RxQaqg5/t3vfqcaqONHBEVXZMNqhxxyiPrVmJyJhMf169eXMEOX7mhQ+OGHH0qjRo3i83HcCDdu27YtlOcEYTM0tj/uuOPUrz5MaHSNhqj4G7+Uw3z8gGyko48+OmFeq1at1FhHEDvOMN8Xt9xyi6o96tu3r8pQuuSSS1RD/NggmFE4B2Y6x4t/kxNVSkpKVAZXWM5JrGD0/fffqx9QsVqjqBw/JYps4QhhhA4dOqi2B+Zf1XjcqVMnCSP8GkLBaNq0aTJr1iyVymyG84EsJvM5QdYGvjjDcE66du0qn3/+uaopiE2oRUE4JfZ3mI8fEEZN7r4BbW+aNm2q/sY1gQ978zlACArtKsJyDpCdZB70EvBDCfd/VM6Bmc7x4l/8aMAPjBh8huCcoW1SWApG6L7ggw8+UN1cmIX9+CkFI8KmTJmiMjImTZqkshGuuuoqo2bNmsbGjRuNMBo8eLBRo0YNY/bs2caGDRvi0+7du+Ovueaaa4wmTZoYs2bNMpYsWWJ06tRJTWFlzlaLwvEjC6egoMC4//77jdWrVxsvv/yycdBBBxkvvfRS/DUPPPCAug+Ki4uNzz77zOjdu7fRvHlz4+effzbC4NJLLzUaNmxoTJ8+3VizZo3x+uuvG4cccohx6623hvYcIEPzk08+URM+9seOHav+jmVj6RzvWWedZRx77LHGwoULjXnz5qmMz379+hm5fvz79u0zzjvvPKNRo0bGp59+mvDZuHfv3lAcP2Uu0oUjePLJJ9WXYaVKlVRq/8cff2yEFT4UUk0vvPBC/DX4MLz22muNgw8+WH1p/va3v1UfElEpHEXh+N966y2jdevW6odBy5YtjWeeeSbheaR233nnnUa9evXUa7p27WqsWrXKCIsdO3ao9xz3fWFhoXHYYYcZt99+e8IXYdjOwYcffpjy3kdBUfd4f/jhB1UYqFatmlFUVGQMHDhQFTpy/fhRQLb6bMRyYTh+ylwe/peqRomIiIgoiiLb5oiIiIgoFRaOiIiIiExYOCIiIiIyYeGIiIiIyISFIyIiIiITFo6IiIiITFg4IiIiIjJh4YiIiIjIhIUjIor717/+JXl5eWqsOSKiqGLhiChkULhJN919990SNK+//rp0795dDfjJwhkR+a3A7x0gIndt2LAh/vcrr7wid911l6xatSo+r1q1ahI0u3btklNPPVWNjH7llVf6vTtEFHGsOSIKmfr168enGjVqqJqY2OO6devK2LFjpVGjRlK5cmVp3769zJgxw3JdpaWlMmjQIGnZsqWsXbtWzSsuLpbjjjtOCgsL5bDDDpNRo0ZJSUlJfBls77nnnpPf/va3ctBBB0mLFi3kzTffTLvPl1xyiSrEdevWzcUzQURkDwtHRBHyxBNPyKOPPiqPPPKIfPbZZ9KjRw8577zzZPXq1eVeu3fvXrnoootUiOuf//ynNGnSRP07YMAAufHGG2XlypUyceJEmTRpktx///0Jy6LAhFogbKNnz57Sv39/2bp1axaPlIjIPhaOiCIEhaLbbrtN+vbtK0cddZQ8+OCDqvbo8ccfT3jdzp07pVevXrJlyxb58MMPpU6dOvFCz7Bhw+TSSy9VtUZnnnmm3HvvvaqQZHbZZZdJv3795IgjjpDRo0er9S1atCirx0pEZBfbHBFFxI4dO2T9+vVyyimnJMzH4+XLlyfMQ8EGobdZs2ZJlSpV4vPxuo8++iihpgihtz179sju3btVGA3atm0bf75q1apSVFQkmzdv9vDoiIjcw8IREZWDUNhLL70kCxYskDPOOCM+HzVAqD06//zzyy2DNkgxFStWTHgO7ZDKyso83msiInewcEQUEai9adCggar56dy5c3w+Hp944okJrx08eLC0bt1atUf6xz/+EX89GmIj8w3hMiKisGLhiChCbrnlFhk5cqQcfvjhqq3RCy+8oBpcv/zyy+Vee/3116uQ2TnnnCPvvPOOSrVHRhkeo3H2hRdeKPn5+SrUtmLFCrnvvvts7xcaayMbDmE/iHU9EMuyIyLKJhaOiCLkhhtukO3bt8sf/vAH1Qbo6KOPVmn2SLdPZejQoSochjAbUv6R3TZ9+nS55557VGNuhM+Q5n/FFVc42i/sw8CBA+OP0WAcUJALYqeVRBRueYZhGH7vBBEREVFQMJWfiIiIyISFIyIiIiITFo6IiIiITFg4IiIiIjJh4YiIiIjIhIUjIiIiIhMWjoiIiIhMWDgiIiIiMmHhiIiIiMiEhSMiIiIiExaOiIiIiExYOCIiIiKSX/0/CaxcHlL8RD0AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkYAAAHqCAYAAADh64FkAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjYsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvq6yFwwAAAAlwSFlzAAAPYQAAD2EBqD+naQAAXvZJREFUeJzt3Qm8lFX5wPHngoDsiQvIIioaKgYKqIAlqChupdbfzEpcKQ1NslJxA5cEwwW3EDOkUtK0P5rmkoJAhibgEmpQuEGyaX9lU7Z75/95jnduc4dZ3uW8++/rZ7zcuTPve+Z935k55znPOacml8vlBAAAANIk6gIAAADEBRUjAACAelSMAAAA6lExAgAAqEfFCAAAoB4VIwAAgHpUjAAAAOpRMQIAAKhHxQgAAKAeFaOMGzJkiLkhHLNmzZKamhrz062xY8ea5xbafffd5cwzz7RYwnTR46XHLWwvv/yyNG/eXN5//33JwrVp03vvvWfKMXXqVGvbfOutt2S77baTN954w9o2kV5UjEL09ttvy/e//33Zc889Zfvtt5d27drJoYceKrfddpt89tlnge1XPxT0y0E/cJBunOt4uOKKK+S0006T7t27m9/r6urMF/3XvvY16datm7Ru3Vr2339/uf7662Xjxo2uKi75W4sWLaRjx46mYXPDDTfIhx9+KGn15JNP+qrg7rfffnL88cfL1VdfbbVcSKftoi5AVvzpT3+SU045xXyYDR8+3Hwobt68WV544QX56U9/Km+++abcc889gX1ZXnPNNeYDVCMMhf785z8Hsk+EY/HixdKkSRNH5xrheO211+S5556TuXPnNtz36aefyllnnSUDBgyQ8847T3bZZRd58cUXZcyYMTJjxgyZOXPmNtHAcn74wx/KQQcdJLW1taYypPvR7dxyyy3y+9//Xo444ghJMq1MakOxWbNmjSpGd911l6/KkR734447zjRQe/ToYam0SCMqRiF499135Vvf+pZ5w+sH4K677trwt5EjR8qSJUtMxSkKGu5HcmlFG/Fy3333yW677WYqQYXvs7/+9a8yaNCghvtGjBhhKq/5ytHQoUMdbf8rX/mK/M///E+j+15//XU5+uij5Rvf+IapHBd+xiSNVhA1om6bHt8ddthBfv3rX8u1115rfftID7rSQvDzn/9c1q9fL7/61a9KfmDttddectFFFzX8vnXrVrnuuutMq0a/+PTD8/LLL5dNmzY1ep7ef8IJJ5io08EHH2w+TLSb7je/+U3DYzR8r5EqdfjhhzeE4fN5BMU5RvlwvbY8f/azn0nXrl3Ndo888khTgXOS31Iqb2n16tVyzjnnmNC/bq9Pnz7mA8pJjkOpnIOVK1eaFriWT4+RHtcTTzyxaheSlrdNmzaydOlSc+z03126dDGtUbVw4ULT4tauDq3ITps2bZttvPPOO+aYdujQQVq1amW+AEtVbP/973/LSSedZLalEYIf/ehH25xD9Ze//MVsT79M9bVoV4s+1kn3auE5qHSuzzjjDNlpp51ky5Yt22xDv1B79uxZcT//+te/zJdup06dzPnT466V/TVr1jSqEOix09eqr0O7LyZNmlSyzHrstVz9+/eXli1bype+9KWG8/6///u/5nfdT79+/eTVV18teQ71PAwbNswc386dO5svu1wuV/WYffDBB3L22Weba1HL2atXL5kyZco2j7vjjjvM3/Qc6xeqlrXU9VDs0UcfNcehMAKkFaPCSlHeySefbH7+4x//ED/0/TRx4kT55JNP5M4776z6eKfXpvrb3/4mxxxzjLRv394ci8GDB5tKXqn8N/2M0PPzhS98wTxe36MaLSv07LPPype//GXzGD2Peu3p51u597tuL//+LOxK1HOt15K+74tp96TuX1MX8jQCpZ9Ljz32WNXjg2wjYhSCxx9/3FRYSn0wlnLuueeaSoO2Cn/84x+bD6Zx48aZD8/p06c3eqx+EOnjtNKhX376Aa8fJPqFoh/qhx12mAm933777ebDZ9999zXPy/8sZ/z48aaL5ic/+Yn58tPK3Xe+8x1TFrf0C14/kLSsF1xwgeyxxx7y8MMPm3LqB3lhpdAp/ZLW7scLL7zQfDhqxUs/cLXCU60LSbsgjj32WHNs9HU98MADplz6JaG5Ifo6v/71r8vdd99tuj0HDhxoyqxWrVplzqN+2Otx3XHHHc250tyRRx55pOGLTl+zVia1PPo4/eL+7W9/ayKGxfRY6PbOP/98sz1N3NUvZf3y0r85Velcn3766abC/Mwzz5hKSWEFU8ukUYtytMtXKyD6xanHWytHWrl44oknzPnTLyCllSC95vRYaKKrXvc/+MEPTH6NRkYL6bXw7W9/23xxffe735WbbrpJvvrVr5pjrmXX5ym97r/5zW9u02Wo51C/rLVSqufw6aefNq9BGxWVogF6/vQ5+sWq53znnXeWp556yrx/1q5dK6NGjTKP++Uvf2mOpb639PrUL9q///3v5vrXcpejx0XPed++fR2csc+Pv9JKq1/5zwHtHtdGTTlurk29T98r+nmix1fPQb4CrBV6bZAV0nOl7xU9b6+88orce++9puJ14403mr/re1avv969e5vzpBVTvRaKK1qF9BpZvny5eX9rOfP0HOq1o+f///7v/0xDJU+vPT2f+vdC+jq0YqR/0xxPoKQcArVmzRptwuZOPPFER49/7bXXzOPPPffcRvf/5Cc/MffPnDmz4b7u3bub++bMmdNw3+rVq3MtWrTI/fjHP2647+GHHzaPe/7557fZ3+DBg80tTx+jj913331zmzZtarj/tttuM/cvXLiw0f7POOOMqtucOHGiee7999/fcN/mzZtzAwcOzLVp0ya3du3aRvsuLue7775r7r/vvvvM7x9//LH5fcKECTm3tLz63BtuuKHhPt1ey5YtczU1NbkHH3yw4f5FixaZx44ZM6bhvlGjRpn7/vKXvzTct27dutwee+yR23333XO1tbWNXvPvf//7hsdt2LAht9dee23zGj/99NNtyjlu3DhTnvfff7/hPi1H8Vu2+ByUO9darq5du+ZOPfXURvffcsstZj/vvPNO2WP26quvmm3qtisp9TqGDRuW23PPPbcps25v7ty5Dfc988wz5j49D4WvefLkydu8nvw5vPDCCxvuq6uryx1//PG55s2b5z788MOG+4vP3znnnJPbddddcx999FGjMn3rW9/KtW/fvuE16Pu1V69eObeee+45s8/HH3/c0eOHDh2aa9eunbkGq8m/Pyqdhz59+uR22GGHittxem3qMd17773NOdR/5+kx0uv9qKOO2ubaPPvssxvt6+STT87tuOOODb/feuut5nGF56hY8ftdjRw5cptrXy1evNjcP2nSpEb3f+1rXzPvx8Jyq2nTppnH/+1vf6t4jJBtdKUFTFsmqm3bto4er0mG6uKLL250v0aOVHGXjXZXaM5BnraANTSt3Qx+aAi8MP8ovw8v29XXpFEGHaVTGNbW1qp2Mc6ePdvV9rTrRcumXS8ff/yxeKFRuTwN6esx04iRtnjz9D79W+Fr1teirWTtCsjT7oDvfe97pgtA8zvyj9PuvcJcEO2G0MeVej15GzZskI8++shEpfR7vbgbyStt6Wsk7I9//KOsW7eu4X6Nlum+8hGxUvIRIY02FXeLlHsdGmXU16HdLnr8Crvc8tetRuLyDjnkEPNTIxHapVh8f6nrTiM+efkIkEa3NPG5FD2ef/jDH0xkSv+t5cvfNCKmZdQoh9LzrhG7efPmiRv/+c9/zE/teqtGR5JpWTU6q/uzQa/FwvNbitNrU5PItQtVI2T6uvLHSq9RjTjNmTPHRAOLE5wL6eeGPjf/OZh/nRq1KX6uF1/84hfNNaLXcZ5GjzQKqNd7cUJ7/rzo6wDKoWIUsHy4ttqHVZ7Oe6JfYpp3VEgrFvqhUjwvSuGXSOGb32uFodx28x8oXrarZd57770bdYWofFeP27leNPyuoXn98NM8kXyXWL5bohrNXdEKZPGXv+bNFH+Q6v2Fr1nLWiofp/i16E89h8XbK/Vc7dLQbkXtCtAvNi2bVihUcYXCD+0W1G6UfHesdk8tWLDAdLNVopUmrahrt4h2+WglQnM+isum3SGa4KoVTL1W9XXkc0eKH1t8feUrX5pfVer+4utOryXtni7+klTl8sx0BJd2/enoTy1b4U0bAkq7ZNWll15qzoVWgvXa1a7ASt09xarlOj300ENy5ZVXmq4v7UItpNdx4c3NVB7a0KjWCHN6bWqlSGkXffHx0mtBu1arndfiz41TTz3VTFGiDRN972qemuYz+qkk6XWt5yb/3tPuZ82lK3Vd58+L0xGAyCYqRiFUjLQP3+3EYk7fuE2bNi15v5MkVL/bLVdGzf/wws32NBfkn//8p8ll0IrOVVddZSonTiIs5V5bUMeyEn1tRx11lIkE6pexJu5qLkU+8dRGq7owSqM5Fvfff7/5XX9q5K0wSlbOzTffbHJstKKjX9Qa7dN8Io2qKB0CrVEEbYnrsHF9Pfo6NKm31OuI4hzky6B5J1q2Ujf90lZ6LWnF8cEHHzTRQY006c9KuVhKc8SqNSB0P/plrvPqaE5VMY3mFN60EuWEVgb0PVHcqPJ7vCZMmFD2eGnl0c3506iiRpo0UqYVF72mtLKk7wGvnxtaudIIdD5qpNe1JsqXaoTkz4uNnC6kF8nXIdBkQ22l6rwlhd0HpehIKP1A0tZaYYK0Jo1qazc/YZwbQbWOtDWoZSqmLbfC1ryWWT8A9XUVRo0WLVrU8Pf89lTxNstFlHTUnnYx6k2P1wEHHGC+wPNf/EHQsuoXZrHi16I/tTKsXwiFx7/4uToKTr/MNIFbvyzz9EvHi2rnWveh0Z8VK1aYEVb65eyk20fpSDG9aaRD587RSoR+seskhZrsqhEE7aorjBo8//zzEgS9lrR7LR8lUnocVbnke410aDRFv4CdDI3XyJd+aetNu+g0IV+TmkePHl12OPk+++zTMEVHKZq8rQn6+sWtkRJNUi9WfO61AuqEJv9rpVUjepU4vTbzc/1o487pVAJO6GeAVqL1ppVo7VLUQQ96rZTbT6XrWiOteh1rxUi7zzR6pCP0StHzovsvvG6AYkSMQnDJJZeYD1kNH2sFp5i2tnX2a6UTkKniN7Z+gCj9AHBL961KVWL80A/Ol156yXxp5OlIpWXLljV6nL4m7RIobPnq6CEdeaUtzny3kX5ga4tTW5SFfvGLXzT6XfNcimcL1rLol165Ice26GvRUWNayc3TnAut+OoXskZl8o/TkTT6ZVVY7uJJPPMt7MKIiP47fz3YPtea56VfMjrSSisWxaN2StH8ED1fhbSCpF8w+eNd6nVoN4uOYApK4bB03a/+rpED/cItRcuooxk1+lMqgls4c3Q+VyhPI2t6bnU/paY8yNOpH7Q7cP78+dv8TUeV6vtXrxN9nxTmZBXSykHhzcmcRDqPkUZRtZJbPAKwmNNrU6OL+r7SEYPaRVfMy0zbmv9TTBs0qtJ7t9p1rdEnze/TyXL1PGsUqRTtOtaKZr6LFiiFiFEI9MNFW+fa8tQoUOHM19ryzg9dz89Hon36+iGlHwJaadAvYo0o6LwjOj+NW/rBox8WmpejX1aao5Ofb8YPrejph6sOm9buGK3gabSmeFZZTeqcPHmyeY36waRfDPq8fMsunxOhH1Y6D49WmPTLW7ejXyD5vI/CyIB++ek+9ctKW92aN6OVznIfiLZcdtll8rvf/c4MYdbuJG2t6rnRlqh+4eYjYjp5n35R67nW16xfbjrUWJNciyMM+jp1WgQd6q2tc92O1xyxaudaoyZ6vvSa0zwgJxVtHbKtic16brSlrZUkfS35ikZ+LiStPGhisw6v1i9SHfKu+9XolG0asdEh+vpe0eRbzTfT7jvt6ivOHyukic4amdDn6DnS60e/rDXpWrt38l/c+no0r0+jYpoLo5UaPZ96vKrl8Oi8Ono9FkZkNMdQIzl6XvXLu3gQhV4D1aLJeTpMXhsGGvnSCpy+jzRSp+8f3a+WuxKn16Zey5pLpNe6ViY0D0srfnqd6jHUa1UjhW7oEH1t+Ohx1IaQvre14aP5fYUDGoppJU3pe06PY3HlR7en3Zh6XWt5S322aYVWB3rkp4IAyop6WFyW/POf/8yNGDHCDCPVYcVt27bNHXroobk77rgjt3HjxobHbdmyJXfNNdeYIbHNmjXLdevWLTd69OhGj8kPe9YhytWGy6tf/vKXZth006ZNGw3JLTdcv3hIcKkhtOrmm2/OdenSxUwRoK9l/vz5Jfe/atWq3FlnnZXbaaedzGv/0pe+tM22lA7j/cY3vpFr1aqVGXb8/e9/P/fGG2802rcOtdbhu/vss0+udevWZpj1IYcc0mj4cTk61FufU+qYlRqeXeoYv/3227n/+Z//yX3hC1/Ibb/99rmDDz4498QTT2zzXB12rsOG9bXo677oootyTz/99DbDz9966y0zbFunLtDH6TXy+uuvb3O8nQzXr3Su8/Q46f3f+973ck7oUH4dht2jRw/zejt06JA7/PDDzdD0Qn/84x9zvXv3No/Ra/zGG2/MTZkyxexLr59Kx1Tp4/S8lrruCqdmyJ9DPQ9HH320Ob4dO3Y0xyc/XUK54fr5a1H3o+8rfX916tQpd+SRR+buueeeRtMEHHbYYWaouV7b+tp/+tOfmuk3qnnllVe2mdIh/zrK3UpNe1Es/97M37TsO++8synnz372MzNVh1NOr838dA1f//rXG46Fnr9vfvObuRkzZmxzbRYPw9frt/D863N0KoTOnTubzwH9edppp5nPxuJjVXjtb9261UzPoK9Xp5co9dX1gx/8wNyvQ/JLeeqpp8zf//Wvfzk+TsimGv1f+WoTgLTRodIafdSWe+FUD0mhkUeNOJbq3okLjWjmJ05EODTRX1cX0G774uiX0mteI3jFk+QCxcgxAjJGu7g0Ob5S1wX80YRizalzOxUFvNGuRe3G167dUpUi7QrVbnldagmohhwjICN06LmODtT8Fk3uZi6X4GgOU+GgBARDc5Q0N0wjiJpvVW55Ic3tLB5AAJRDxQjICB2RpqMAdVJBElCRBjoSTYfoa7K1rhGYH+EG+EFXGpARmk6oo6N0pFGp+XOSQie/jHN+EcKji1Prda0jUguXiEE8jB8/3kSm84szl6OjCXWEro421alA8ktjRYWKEQAAsGrevHlmmpbevXtXfJxOWaPRbI1k68oFmiSvN7erRdjEqDQAAGDN+vXrpW/fvmaOKp0ZX7s4y81GrvP76SS5mhyfN2DAAPOcUkvmhCG58XSPywjojK86QRuJpwCAMLuxdQqH4sW0gxihZzvxP1e0fIzSyWP1VorOvq6TburM7VoxqkRXEdBligrpJJ66bmRUMlUx0kpR8erdAACEQZdL0lm+g6wU7dG9jaxc7W1B3nLatGmzTV6fLqg8duzYkqNfdSZ57UpzQued0tnlC+nven9UMlUxyk/l/2U5TraTZlEXByH47Kv9zc+Wj2+7dhUAhGGrbJEX5Mmqy8n4pZEirRS9v2B3adfWTmRq7bo66d7vPVOp02Vg8kpFi/QxOmWCLoRcbqHlJMhUxSgfCtRK0XY1VIyyoO0Tr3/+D843gKjUZ/KGlcLRpm2NudlQJ59vRytFhRWjUnTtPZ1bSvOL8nRNP51lX9fn04WC8wtO5+nafsWLq+vv1db8C1KmKkYAAKRdba5OanP2tuVmKZyFCxc2uk8XH9ah+Jdeeuk2lSKliyfPmDGj0ZB+jTg5XVQ5CFSMACABPjv5EPOz5fS/RV0UoCTtKtx///0b3de6dWvZcccdG+4fPny4dOnSRcaNG2d+1663wYMHy80332wStjVHaf78+XLPPfdIVJjHCACAFKmTnNWbTUuXLpUVK1Y0/D5o0CCZNm2aqQj16dPHLO+iI9KKK1hhytQ8RmvXrpX27dvLEDmRHCPAZSQi/7jCxxLFiJ9S5ymofUR13ov3H3V5qtma2yKz5DFZs2ZN1TwdG99xKxfvZjX5ulPPpYGXPU7oSgMAIEXqzH/2tpU1RIyQCWG0ooPYVxAt4bi3rhEfYVwrSb0e3ZQ77IjRskVdrEaMuu3zQaYiRuQYAQAA1KMrDbDccrXZ8i2XQ+GnrF7LF2bUDfHAeU7msbGZNF1nOfk6CagYAQCQIlqZqaVi5BkVIyRaYRSjUiuu1P3FUZc4twD9RIq8RpeSmvuRdcXviVLCPKfVop3lylLpdbgZQenm8YWP4/rPLipGAACkCF1p/lAxQqLFqTVno4VZbRvF9zvJ+/FanuLnxelYo7y4nie3749KUV4vz3XyeC95dG9PHGB+9hj1kqt9Ir6oGAEAkCK1uZy52dpW1lAxAgAgRXRKRnsTPGYPEzwiEeIw0ZzbhM64KJf8mpeUZRVQmZPzZ/sc25jCIYr3dthLioQ9weOif3SUtpYmeFy3rk722XdVpiZ4JGIEAECK1Focrl9L8jUQT+WSjm0mOvtJCnW7L1u8JF8HcSwRHKfnx0kCse1IkZPt2by+vG6r2nFI27Vfm/v8ZmtbWcOSIAAAAPXIMUKiJC264TRvqdrrKR4SnLTjgHQqFZ3ye22m8doOO8fotbd2sZpjdMB+q8kxAgAAyVQnNVIrNda2lTVUjJAo1fJjwmxtOtmX07yQaoonjws7tyNO+4K9UWhOz1u5x5V6XhQjR6P4HOCaTy8qRgAApEhd7vObrW1lDTlGgEte5jPy2zL3+jjbzwWCvt7SeH2GnWM0/82O0sZSjtH6dXXSvxfzGAEAgISqtZhjVJvBHKPERozGjx8vo0ePlosuukgmTpzo6DlEjNIvitmp09LCTerM3rB/bablmnYqbTNfz31zV6sRo0G9VmQqYpTIeYzmzZsnkydPlt69e0ddFAAAkCKJ60pbv369fOc735Ff/vKXcv3110ddHMRMFC1cN/sMMoeoeK4jt7ISHUg7G3MLOX1e/przc91FKa2RsbpcjbnZ2lbWJC5iNHLkSDn++ONl6NChVR+7adMmE1osvAEAkIUcI1u3rElUxOjBBx+UV155xXSlOTFu3Di55pprAi8XkrWivO3WoZtITZDrsdlusae1NZ12NucWqva8zrNzrt57fspSah9+txm3+cAQD4mJGC1btswkWj/wwAOy/fbbO3qOJmdrwlj+ptsAACDNaqWJ1VvWJGZU2qOPPionn3yyNG3atOG+2tpaqampkSZNmphus8K/lcKotPRK+yzPxS3kPFqxyMp7IOzy2Nxm2KPSZr7RzeqotCP2X5apUWmJ6Uo78sgjZeHChY3uO+uss2SfffaRSy+9tGqlCACALMhZTL7OZTD5OjEVo7Zt28r+++/f6L7WrVvLjjvuuM39yJ64tFyTNPLNxvPjFjnIsjDPQRi5e0HkGLm5XpN8TTPBoz/Z6zwEAABIeo6RDeQYISmiWC0ciDKXJy+N13jYOUZP/X0PaW0px2jDujo5tve75BgBAIBkqpMaqbPUIVQnmYmdNKBihFRJSmSlWjmL77c590vcjw3C43VGbJv8bJO14BAEKkYAAKQIydf+UDFCqgTZAnSaC1FqziHbEaBKLd6gWsNRjUCCfbaukSDWSrMxcsxtRDZtanNNzM3OtnKSNYxKAwAAqEfECKkS5AiXKPMwbOyj+DlxyC+BfU4ie9VGO1a7NvJ/9xIlshHN8Vpum+Kcp/R58rWdLrC6DHalETECAACoxzxGQIAtwTi2KuNYJoQbHSWyUnm+sML7yj02zvMYPfz6PtKqrZ1lsj5dVyun9FmUqXmMiBgBAJDC5GtbN6cmTZokvXv3NhUovQ0cOFCeeuqpso+fOnWqWQi+8Lb99ttL1MgxQir5ad1FkZcUZuu72nPj3MLHtiqdp3LnMohrwOm+gry+nG7by3Hh/VBd165dZfz48bL33nuLdkb9+te/lhNPPFFeffVV6dWrV8nnaAVq8eLFDb9r5ShqVIwAAEgRnfU6ipmvv/rVrzb6/Wc/+5mJIr300ktlK0ZaEerUqZPECRUjJIrT0ShxzbdIwjxNSTgOCPY8BRlpdRrN8VMWv2WwSV/P1i0bRR5/LLR91uZqzM3Wtryora2Vhx9+WDZs2GC61MpZv369dO/eXerq6qRv375yww03lK1EhYWKEQAAqJrYXahFixbmVmzhwoWmIrRx40Zp06aNTJ8+Xfbbbz8ppWfPnjJlyhSTl6TJ3TfddJMMGjRI3nzzTdMtFxVGpQEBRlTyMwNXm+8lyFXLq81ZY+s1INujKcOIOJaLJtm61oMS9qi0qa/2sToq7cwDX9/m/jFjxsjYsWO3uX/z5s2ydOlS81ofeeQRuffee2X27NllK0eFtmzZIvvuu6+cdtppct1110lUiBgBAJAidbkm5mZnWznzc9myZY0qdaWiRap58+ay1157mX/369dP5s2bJ7fddptMnjy56r6aNWsmBx54oCxZskSiRMUIcMhL69NWpMhLC7jaiBunkSAiRfEWRHTEyxpkYaxw73Qm7yDLkFXt6ofgu6W5Q5s2bXKcl6Rdcccdd5xEiYoRAAApUitNzM3OtnKOHzt69Gg59thjZbfddpN169bJtGnTZNasWfLMM8+Yvw8fPly6dOki48aNM79fe+21MmDAABNh+uSTT2TChAny/vvvy7nnnitRomIEFAlznhWvI3kqldFp+YkEJZOfEZh+xXmOLy/bJFJk1+rVq03lZ8WKFSbXSZOqtVJ01FFHmb9r7lGTJv+tsH388ccyYsQIWblypeywww6m623u3LmO8pGCRMUIAIAUqfMxzL7Utpz61a9+VfHvGj0qdOutt5pb3FAxAkLIlbC9GriTGXrJn0gnL+czTqO2bOy72jbCmAspOxM8NpGsyd4rBgAAKIOIEVKt1CiVoNduKrVCd7XRNEHwMrIIyePkfDo912FEloLM3QtyH0l6H7ld/LXatrKGihEAAClSJzXmZmtbWcPM10BIeUFAVKK4Zr3uM4xZ4INQaR9hz3x9+4IB0rKNnbjHZ+u3yg/7vRR42eOEiBEAAClCV5o/VIyAMmzmBdlqPdtcr42IWHZ4PZd+roUgrqsgRnECxagYAQCQInZnvm4iWUOOEVIhSTkUtreBbHEzh0+5qM3ywTXWZz/3OzIs6jygIIWdY/TzeV+xmmN0yUF/yVSOUfaqggAAAGXQlYZUcLo6faXHum1NllvR2836ZV7nl7H1WCSPn/Oaf26P6cFdO16v6TCu7ay8J3S2altdYHUZjJ9QMQIAIEXqck3Mzda2soYcIyDGORBxj/7EvXzItriMvAw7x+iGlw+X7S3lGG1cv1UuP/j5TOUYETECACBFaqXG3GxtK2uoGAEOhdG69JqDFJW4lw/+RrYl/fyGsUZhHI8VXWn+ZO8VAwAAlEHECIhxizGOrVEkX7XrKeqcnKj26UUcy1drsQusVrKHiBEAAEA9IkZAjFuMcWiNJqXlnnZpido43WZSrrc4vj/IMfKHihEAAClSm2tibra2lTVUjIAEtJ6LvT1xgPW1rsqJU0s4y8LI+/EyG7VbcZ6jy8kM+cXiVH7YQcUIAIAUyUmN1FlKvs4xjxEAW2tAlXqOrRZ5PlJUal9pyUVBZUFGc8JgY1+2r7u0rEVIV5o/2XvFAAAAZRAxAiyr1JL028p0EnGKoiWb9hmU08breSmMUHp5vpsyOCmj1/L7iQwl4Vquy9WYm61tZQ0VIwAAUqRWmpibrW1lDRUjZE6Soxheyhzl603iMc4Cr/loSctBCmKbSf78gDNUjAAASBG60vyhYoRMyM/7ozrPzllpESal5ZiW15FVQZ6fKM953KM35bbJ+yT9qBgBAJAiddLE3GxtK2uoGCFVyrUg3cwQ7TXfwkvr1e3InEqP9zpiiBawpCLi5+TxNkaCuXlc0NuwPXeXl5mv46g2V2NutraVNdmrCgIAAJRRk8vlKidcpMjatWulffv2MkROlO1qmkVdHIQgjPyZOK2hlvR9I7mieK8VR8uqRZLKldFJJNZPhHhrbovMksdkzZo10q5dOwn6O+77c74hLdrY+Y7btH6LTD7sD4GXPU7oSgMAIEVyuSZSZ2kpj1wGlwShYoRUs5EDYWNbbnnJAbHVYidSlExRR/rC2K/bWaidPr5UBMrveoZILipGAACkSK3UmJutbWUNFSOgDJujuoKK5vgZRWNrhBLiISnnKU7XnZO1B91uA8lHxQgAgBSpy9mbsbouM8Oz/ouKEVItzBFjlfZlu3VsI9fI79+zKI7RgTiWqZJqI8TikMsXVHQpLHUWk6/rMph8nb1XDAAAUAbzGAEx5mdkXBS5HEmLXqRJlo590l5r2PMYnf78adK8TXMr29y8frP89vDfOSr7pEmTzO29994zv/fq1UuuvvpqOfbYY8s+5+GHH5arrrrKPGfvvfeWG2+8UY477jiJEhEjAABSJL8kiK2bU127dpXx48fLggULZP78+XLEEUfIiSeeKG+++WbJx8+dO1dOO+00Oeecc+TVV1+Vk046ydzeeOMNiRIRI0Qmbq2+uJUnDa8v7cc0aaIc8eX1caVmt7ZV/rcnDjA/O8/ObbNdm8cq7IjRt2d+22rEaNoR0zyXvUOHDjJhwgRT+Sl26qmnyoYNG+SJJ55ouG/AgAFywAEHyN133y1RIWIEAECK5JOvbd3yla7C26ZNm6SS2tpaefDBB03FZ+DAgSUf8+KLL8rQoUMb3Tds2DBzf5QYlYbIRB1FcDvKxG0L2MljvXJSljiMorGxzzhFncJcfT6Ibdsqj80Z2L2MjrR1fHuMeslzueKsTmrsDdeXz7fTrVu3RvePGTNGxo4du83jFy5caCpCGzdulDZt2sj06dNlv/32K7ntlStXSseOHRvdp7/r/VGiYgQAACpatmxZo660Fi1alHxcz5495bXXXjNdb4888oicccYZMnv27LKVoziiYhSCOLV4k9p6tnkMvW7L5kzYQawuHuV8MEGK0+uIwzUQ5HvAxhxYYZyv5YM/j2J0lnR+tvqV04iRpaU8cvXb0UqRkxyj5s2by1577WX+3a9fP5k3b57cdtttMnny5G0e26lTJ1m1alWj+/R3vT9K5BgBAIBA1NXVlc1H0i63GTNmNLrv2WefLZuTFBYiRgGIQ25HGBGYIFrP1WbDDbPcYfCbjxH3tZ2SFi31M29Ukl5/tVXm465SblClyGpSXp9fml9kb0mQGsePHT16tJmzaLfddpN169bJtGnTZNasWfLMM8+Yvw8fPly6dOki48aNM79fdNFFMnjwYLn55pvl+OOPN8naOsz/nnvukShRMQIAIEWiWhJk9erVpvKzYsUKM21A7969TaXoqKOOMn9funSpNGny3+0NGjTIVJ6uvPJKufzyy80Ej48++qjsv//+EiXmMfIpjBFIXgSxBpctSWm9xWEEUhSS9nqSVl6Ed67jcm2EPY/Ryc+eJc1a25nHaMuGzTL9qPsCL3ucEDECACBFoupKSwsqRgAApEidxVFpdZa2kyRUjEIUZljXa6jZzYSBbv/utmxRczqUOSmvJ67dDH7L6XXIOZLH7bnmGoAXVIwAAEgRutL8Ifk6BHFqwYaxVEFeHF5vlsXpugtDqcVGbU1pEIdjGYcypEXYk86GnXx97NMjrCZfP3XMLzOVfJ2YCR513oODDjpI2rZtK7vssoucdNJJsnjx4qiLBQAAUiQxXWm61srIkSNN5Wjr1q1mzoOjjz5a3nrrLWndunUsWxhxnEgv7ksV5NE69s/2sXMyNUWU17zNpTKKxeE6jEMZ0iLtx5KutIxUjJ5++ulGv0+dOtVEjhYsWCCHHXZYZOUCAADpkZiKUTHt71QdOnQo+xhdn6VwjRbtf/UqTlEfv8+p9LyoIzVR7z+LnF4LhX8PaoSem/PvNf+n0uO57uBUnPLPihExykiOUfGidKNGjZJDDz204tThmpekiWj5W7du3UItJwAAYcsVzGXk95aT7EnkqLTzzz9fnnrqKXnhhReka9euriJGWjkKe1Sa29FbQYzu8tqqLnxOUqfjR/yXS3FzrfidT8vJvqIYYcn7Jb3CHpU29Mnvy3atW1jZ5tYNm+S54yZnalRa4rrSLrjgAnniiSdkzpw5FStFqkWLFuYGAEBW0JWWkYiRFvPCCy+U6dOny6xZs8wqvGHMY/T2xAHmZ49RLzlu3SV58dE4lgnZFuU1yfsBSYwYDXnifKsRo1knTCJiFEc6VH/atGny2GOPmbmMVq5cae7Xi6Bly5ZRFw8AAKRAYipGkyZNMj+HDBnS6P777rtPzjzzzMD2WxwpyqvUgrS9xpabnIhy+UBxnFMJ8RTE+fOzzSivozDfP7xvYAtdaRmpGCWkxw8AACRYYipGceVkNmC/3MzpUu33MKIBQY5aYz224AV5TMvl7DnhdbSZjehpGNcZ1zJsIWLkDxUjAABSJJerMTdb28qaxIxKs8HLqLS05BKQY4Q4iFv+UrVt5nmZ/6vc8+I66zzSMyrt0McusDoq7a8n3smoNAAAkEz5WattbStrqBgFoFrLL4qZo72upeblubR4UY6fKInTXKEoc4q85AMW30/kCH6RY5TBtdIAAACCQMQoAF5bel5HdwXBz75p8cKp5YM/b432mF49Yul0HrBqj7NxfVaLVlUrY6Vt8b6BXyRf+0PFCACAFKErzR8qRj55ycXxOhN2GHMm2RDnssH/HERe5yGyNbO8X05GilWL/NiIQhFZBeKJihEAAClCV5o/VIx8CnPtp6TMQRTE7NRRrv0Gu1Eiv4I891637XY2bi/7ABAOKkYAAKSIRnls5QbliBihmnwLsGE0TYit6Lisy+Z2G05H8FTav9t8LMSbn6hPEOc+iLmQ3D6eKChs0eUsbK1pkZPsYR4jAACAekSMXMq35ornXQlz32mRttcD51GPuJ97pxFKm6PR4n5MkBy6jIf+Z2tbWUPFCACAFGFUmj9UjFwiD8A9P61tjneyOJ2LKy3n02aOVNqODZBUVIwAAEgRHZFWw8zXnlExcsnL2kdeVdqO033EuRVaapZh8i7SLW3n00+OUV7xGmlxfs8CWUDFCACAFNGh+taG6+ckc6gYWeS1pedlzhSn+whiZfFi1bZZ7e+0jNPD1nsgKWzOeJ3UY4D4IfnaH+YxAgAAqEfEyKJqMzuHOauz17lSvLRavbaA86u1x2UNLviXP9duz20Q+TVRrHhvI+cI8IuIkT9EjAAASBEdSWbz5tS4cePkoIMOkrZt28ouu+wiJ510kixevLjic6ZOnSo1NTWNbttvv71EiYiRT25GVIWZQ+B1H5VGirnNhapWBiJF6VXu3FaLJNl8bziNjhY+1u/+q0VPSz2m1Huu8HdyjpAUs2fPlpEjR5rK0datW+Xyyy+Xo48+Wt566y1p3bp12ee1a9euUQVKK0dRomIEAECKRDUq7emnn94mGqSRowULFshhhx1W9nlaEerUqZPEBRWjAHN2ip8ThzmHnOZ+uBlNAzi9hoOIEnp93/jJPXK7TyePYw4v2K0Y2coxEs/WrFljfnbo0KHi49avXy/du3eXuro66du3r9xwww3Sq1cviQo5RgAAoKK1a9c2um3atKni47WSM2rUKDn00ENl//33L/u4nj17ypQpU+Sxxx6T+++/3zxv0KBB8u9//1uiUpPLZWf6Jj2Z7du3lyFyomxX08zKNivlDqQNOQ/wOodVsSDm0XK67TBGvvnZB++z9Nma2yKz5DETQdF8mqC/4/b67Whp2spOAnPtpxtlyenjtrl/zJgxMnbs2LLPO//88+Wpp56SF154Qbp27ep4f1u2bJF9991XTjvtNLnuuuskCnSlAQCAipYtW9aoUteiRYuyj73gggvkiSeekDlz5riqFKlmzZrJgQceKEuWLJGoUDEq4id3oFrrsdRzbJYlaKzlBLeCvEbCyC3yui0nuYflHsv7Cn5pN5CtrqBc/U+tFFWLdmkH1IUXXijTp0+XWbNmyR577OF6f7W1tbJw4UI57rjjJCpUjAAASJGoJngcOXKkTJs2zeQL6VxGK1euNPdr917Lli3Nv4cPHy5dunQxcx6pa6+9VgYMGCB77bWXfPLJJzJhwgR5//335dxzz5WoUDEq4ieaU631mH9OlBEXp/v2s+J9mK+P6FU0yo1uLHd/EOcpzNmpg5i13W/5/W4HsG3SpEnm55AhQxrdf99998mZZ55p/r106VJp0uS/474+/vhjGTFihKlE7bDDDtKvXz+ZO3eu7LfffhIVKkYAAKRJEH1pDjgZy6VdbIVuvfVWc4sTRqUlKIrhZDbqoMtgU6kysn5aNketxeU6jVt54lomxHtU2p5Tr5Amlkal1X26Ud4582eBlz1OmMcIAACgHl1pAfDbsiuXQ+BmNuokjCArtUZUGLkp8M9rpLI4Iuhn9ukgeI28Blk2rn0kZUmQtKBiBABAikQ1Ki0tyDHyqFQL0W+rMcn5QkCW8J5EnHOMdp9ypdUco/fOvj5TOUZEjAAASBON8tiK9OSyFzGiYuRSubWf/OQheGl9RjGXCa1kFHM7ijAO11AYa6XZFIdjBmQJFSMAAFKE5Gt/yDGy1GK2OaIqKS3E4nImpdxIB6dR06RFiKLYF9KVY9T9l1dZzTF6f8R1mcoxYh4jAACAenSl+WzNVcqt8Lrump8Woq1WZqW10srtw2bLlmhUOtic+drr+8Tm+8nmiNFqz/WzbiOyjeH6/lAxAgAgbTKTJGMfFSOPol4l3FbUJozoTxDloZUcT27PS7VIUuFjbM0P5mYOsmr79HP92bp2eQ8AdlExAgAgRehK84eKUUIF3dqslGNku4VaaV/l0EqOJ7+5REFEAp1cv0FHJL1c4wCiQcUIAIC05RfZyjHKSeZQMcrYzMNxzDGKwyg82Ln+/M7IXunxNke4uS2H01GSft5PXMuwR7u/bHWB1UjWMI8RAABAPSJGIQqiReh3jao4jPbyk39B6zpa5a6/IM5LtW0uH/x5y7bH9MrbsXGN+517yHZ5gEboSvOFihEAAGlCxcgX1kqz2LoLaiRLEC1JWqlIcm5cUCPdeF8gDWuldfvFWGnS0tJaaZ9tlGU/GJuptdKIGAEAkCY695Ct+Ydy2Uu+pmLkUr4l+czy183PYZ37bPM3t/Kt1IYcifpWtJ8ROtVEkVsURMudFn6y5M9X59k533lnQc55FeX153dkH6D9QLb6gnKZ6VP6L0alAQAA1CPHyCUbLcSktwiJ0qCcKEc1sho94irsHKOud1xjNcfo3xeOyVSOEREjAACAeuQYuRTErLpJa9FWm/0X6Wd7Xiw/11I+N6+zlJ6durhs5f7upfxxys3j/YisJl/vueeeMm/ePNlxxx0b3f/JJ59I37595Z133nG1PSpGAACkSE3u85utbcXde++9J7W1tdvcv2nTJvnggw9cb4+KUQBsjxizwelaT06eG2Q5kQx+oxc2oxvVZt9umDOpaEbsuM/u7vYY8X5E1vzxj39s+Pczzzxj8qvytKI0Y8YM2X333V1vl4oRAABpkpGZr0866STzs6amRs4444xGf2vWrJmpFN18882ut0vFKABuW8lu73ei+Ll+oj1+W6LkPqSf3+iGm2vD7ajO4jmT3JTVb8TLy7VfKvcJcCUjOUZ1dXXm5x577GFyjHbaaScr26ViBAAAEuvdd9+1uj3mMYpwZXgn2yy33fxjelz6D/Nz+YC1je532tKtFq2qtC0gSdHDMNYezOM9g0jXSrvlOrtrpV18VeznMdJ8Ir2tXr26IZKUN2XKFFfbImIEAECaZCTHKO+aa66Ra6+9Vvr37y+77rqryTnyg4qRT0G0DIvXTCvV0s3/e3mZkTZOR5L5WYfKawvcyfPIS0ontyOy3DwnDteQjX01jKKr8P4H8F933323TJ06VU4//XSxgYoRAABpkrGI0ebNm2XQoEHWtkeOkUdxacWFWY64vGakX5Zy3HhfpV/oOUY3Wc4x+km8c4wuvfRSadOmjVx11VVWtkfECACANMnIcP28jRs3yj333CPPPfec9O7d28xhVOiWW24RN6gYRcDmauBhjoSjRZttlaI4tq7p4vyaqAUxX1G1OcYQvLRH6bK2JMjf//53OeCAA8y/33jjjUZ/85KITcUIAAAk1vPPP291e55yjObPny8/+tGPpGnTpnLJJZfIcccdZ+4/+eSTZfr0omFSKZ/HKKj1yZJcljhuO0s5K1FKe0scSEKO0W43Xm81x2jppVc6Kvu4cePkf//3f2XRokXSsmVLkxB94403Ss+ePSs+7+GHHzb5QboY7N57722ek69XRMFTxOj888+X6667zvz7sssukyeeeELuuOMO+eSTTyRod911l0yYMEFWrlwpffr0Mfs9+OCDA98vAAAob/bs2TJy5Eg56KCDZOvWrXL55ZfL0UcfLW+99Za0bt265HPmzp0rp512mqlUnXDCCTJt2jSzBtorr7wi+++/vzhx+OGHV+wymzlzpgReMdKa4DHHHGP+fcQRR8gPf/hDOfbYY+XTTz+VID300ENy8cUXmzkLDjnkEJk4caIMGzZMFi9eLLvsskug+wYAAOU9/fTTjX7XuYX0u3nBggVy2GGHlXzObbfdZuoTP/3pT83vGnR59tln5c477zTf9U7k84vytmzZIq+99prJNypeXDawilGTJk1MxKZTp07SvHlzU/jbb79dfvzjH0uQNLN8xIgRctZZZ5nfdb9/+tOfzHTfGrmKGz8LZdridKLHIPYVRLeK120lpWsnjK6oagnOdIeFI+3H2e/r85PQnnUaO7GWfC3eafeb6tChQ9nHvPjiiybgUUgDHo8++qjj/dx6660l7x87dqysX79eAq8YaZKTVka0QlRIo0br1q2TICdw0lrn6NGjG1XQhg4dag4sAAAILn+pUIsWLcytHF2vbNSoUXLooYdW7BLTIEvHjh0b3ae/6/1+ffe73zWpNjfddFOwFSMNeWkl6IYbbmi476OPPjJRnBdeeEGuuOIKCYLuo7a2tuQB1ESvUjZt2mRu5U5sFkS5HEKUrbcoWpCV9um0PGGUt9pQeD9D8ZPeYrd13TjZjtOFnpPKb/ndPD9OAz3SOo9Rt27dGt09ZswYE5EpR3ONtCtL6wVR0aDJ9ttvH07EaPjw4aYPUJOk3n33XTnnnHPki1/8ounTixNN5tLF5QAAyIwAlgRZtmxZo1FplaJFF1xwgRmUNWfOHOnatWvFzWtKzqpVqxrdp7/r/U59/etfb1zkXE5WrFhhRtB7mQ3b03B97bM777zz5JFHHjHhMk2W0mH7fle0rdaV1qpVK7NPzVjP08QqHQ332GOPOYoYaa3X5nB9hoEj7teEzQlFgSSL6toOe7h+93E/kyYeIiWl1G3cKO+PvsJR2bU6ceGFF5ppe2bNmmWG3ldz6qmnmoFbjz/+eMN9OsxfZ7B2mnydzzsuTLPZeeedzeAwHRUXSvL1P//5T1MT05rg8uXLzagwfWHlhuPZoDlN/fr1kxkzZjRUjLRSpr9r7bSUan2gAACkTkSLyI4cOdL0JGmgom3btg15QlpZ09HsSnucunTpYnp01EUXXSSDBw+Wm2++WY4//nh58MEHTf1Cl/hw6r777hObXFeMxo8fb/oWv/e975n5hJYsWSKnn366qd3df//9MnDgQAmKZq5rhKh///4moUqH62/YsGGb2mKY6NtOD7+TRzpZ4oFFf5FENq8lJ8sPed1mHHL4srwkyKRJk8zPIUOGbFNxOfPMM82/ly5daiI6hdEhrUxdeeWVZt4jjTLpiDSncxgV0gFa//jHP8y/e/XqJQceeKCEUjHSOQe00DpvkdLCv/zyy+YF6cEo7LqyTUNuH374oVx99dWmJqpzF+i8CcUJ2QAAIFw5B5k52sVW7JRTTjE3r1avXi3f+ta3zLa/8IUvmPs0xUYnftQIlHarBZpjpKPDdtppp7KzXmpILK7CXhIE7qUtymFjCRMvz/ezb2RX2q6ZuLyesHOMdr/ebo7Re1c6yzGKigZN3nnnHfnNb34j++67r7lPZ9vWHqa99tpLfve73wUbMSpXKVJxrhQBAJAJEeUYRUV7jp577rmGSpHab7/9zBJioSVfA0Eh96axamWsNALOb64UssfPHFZxFGb+EqKjA7GaNdu2F0jv07+59d8MKAAAkJrka1u3uNNh+Tq6TUfJ533wwQfyox/9SI488kjX2yNihFQr1dpzGoWxEXGJw9puxeUqXiutOJ8JyVYqiuj1mk5blMTLcUjbMUijO++8U772ta/J7rvv3jBDt05IqYPDdLS8W1SMAABIkwCWBIkzrQy98sorJs8ov0SY5hvpWqpeeJr5OqkYlYYs5BqUK6fNSBjC4+f4Fz/X9rUR1ZqDeUGW1+YxCXtU2h5jb7A6Ku3dsZfHclTazJkzzQTPL7300jZl0/LqHEk6e/ZXvvIVV9slxwgAACTOxIkTZcSIESUrbFpB/P73vy+33HKL6+3SlYbMq9YyTFoOzvLBn4e+e0x3FzVAPPk5T05HKsbpWqhUljAjXXE6JkmZ+Tpsr7/+utx4441l/65D9W+66SbX26ViBABAmmRkHqNVq1aVHKaft91225nVMtyiYoRU8NNyrBYp8jNrtd9WZ/EIMieKH+s0akBkKX38nsMwroEg3rth7BvR08Vo33jjDTO7dSl///vfZdddd3W9XXKMAABIE5tzGOUkto477ji56qqrZOPGjdv87bPPPjML3p9wwgmut8uoNKQSLcH/4lggCTl8ab4+wx6VtueVN0hTS6PSajdulHeuj+eoNO1K69u3rzRt2tSMTuvZs6e5X4fs63IgtbW1Zhi/24Xm6UoDAACJ07FjR5k7d66cf/75Mnr0aMnHeWpqamTYsGGmcuS2UqSoGCGV4tL6jOO8MVlooSM+10i1bUSRv1Tt/qDLFbiMJF+r7t27y5NPPikff/yxLFmyxFSO9t57b9lhhx3EKypGAAAg0XbYYQc56KCDrGyLihEyl68Q5uy55crldJ82yup0VFq5fSe65QxP5zSMc+5lxKXbEZRu521Ky7WelXmMgsKoNAAAgHpEjJAoNlq6ttcKcxNZcRsBsrE+lp+WOZKt2qztUUZPncpfv4XXsNcoKOAEFSMAANIkQ8nXQaBiBFThJ6rTsG6Zw2iN27yeUqNoip9bbd9uV2B3Uz6Er1RExm9Ojc2oY36f+bX8qrEZ6cxK3hw5Rv6QYwQAAFCPiBFSzU3rOQhuW7tRrFOWxBXYUZ6bXDenUcK456f5HW1XKe8psdd9BiM9thAxAgAAqEfECIhRPoLbfUZRtrD3C3v8zvcT5HvCz7b9lid11zPJ175QMQIAIEVIvvaHihFSzcY8QNXut7ktGy3XoFr1qWtVZ5DfObyCvAa4vhAXVIwAAEgTutJ8oWKEzKkWUSk3UsdPizaolreTeYyKMRM2ooji2JxB3va+04auNH8YlQYAAFCPiBEyx2mL1e2cQqUeF3QLtdL2y62D1Xl2LtOt6bQK8zza2Jfb91Uht+/FIMoZ6/cNXWm+UDECACBNqBj5QsUIqeJktXC/c7lEyU0rNQmvB/a4udajUO39ZvN69RLddbutOBxTBIOKEQAAKULytT9UjJAqXlqATqJMtvbtVhjzyEQxSgjh8JoX43bkppdtRnl9xaEMiC8qRgAApAk5Rr7U5HK5zLzstWvXSvv27WWInCjb1TSLujgIUZjrfAW5L1q4iFLSrj9b0WC/tua2yCx5TNasWSPt2rUL/Duu50U3SNMW21vZZu2mjbL4tssDL3ucMI8RAABAPbrSgAha015b3jbXfqv2uLi0tuGd2+vMaW5RGGWJa05REqJmJF/7Q8UIAIA0IcfIF3KMkCqlZsktbt0locUXpxE8SAY310jW1stz+n7ykx9Y6fiHnWO0z4V2c4wW3eE8x2jOnDkyYcIEWbBggaxYsUKmT58uJ510UtnHz5o1Sw4//PBt7tfndurUSaJAjhEAACmS70qzdXNjw4YN0qdPH7nrrrtcPW/x4sWmMpS/7bLLLhIVutKQKn5msi0WZY6N01mBSyG6lE1uZr7OR4qCuMajXFus2qz2bnKoyh2bcvfH6v0WYVfasccea25uaUXoC1/4gsQBESMAAFC1m25twW3Tpk1Wt3/AAQfIrrvuKkcddZT89a9/lSgRMULmOG25xqoF6EJSyw37wrzGgxzRFvTrLVX2RH8+BBAx6tatW6O7x4wZI2PHjvW9ea0M3X333dK/f39T2br33ntlyJAh8re//U369u0rUaBiBAAAKlq2bFmj5OsWLVpY2W7Pnj3NLW/QoEHy9ttvy6233iq//e1vJQpUjJA5tlp6XvIzwsz/qbYvW+tmIX28nHO/10eQcwz5iWYlcT6vmvqbrW0prRSFNfP1wQcfLC+88IJEhYoRAABpkvB5jF577TXTxRYVKkaAR+VGqTh5ThitZ1ujgpLQQkZpSTrnNvbpNiLk5PhEMbouydavXy9Llixp+P3dd981FZ0OHTrIbrvtJqNHj5YPPvhAfvOb35i/T5w4UfbYYw/p1auXbNy40eQYzZw5U/785z9H9hqoGAEAkCJRLgkyf/78RhM2XnzxxebnGWecIVOnTjVzFC1durTh75s3b5Yf//jHprLUqlUr6d27tzz33HMlJ30MCzNfI5Vo1SVj3hkkU6VrI065a25nvA6qLGHPfN3r+3Znvn5zsvOZr9OAeYwAAADq0ZWGVPHS8otzZMTNiBjb+SRxPB7wx++17iUnp9w2glBcPqczyKfyWs9MX5B9RIwAAADqETFCqpTLJai0anacW4thziOD9KkWQXHLTyQ2jMgs74Hok6/TgIoRAABpkvB5jKJGxQipUi5/IYoVvcN+nWHOkQRUk6TIbNwiX4gWFSMAAFKErjR/qBghVWyuV+Z0RJiXlqPfdcz8jE6jpZsdttcvc5O753Sbbh4XRbQmke8XutJ8YVQaAABAPSJGSAU3LUmnuQI2Worlok7VZt61uUZauX04RU5FdlSbYyh/Dbw9cYD52WPUS6GUq3j/QV27pSJgtkf2hYGuNH+oGAEAkCZ0pflCxQipVqnFaDuyUorTbYTRCvW6jyS0kCFWruVykczi3/ORIi85RmG+J/xsg0hpdlExAgAgTYgY+ULFCKngJk/IaW5REC3GqHIznKCFnD62Z7q2MTdRuby7IK8/v6NAkS1UjAAASBGSr/2hYoTMqTYiLMjWYxwjRXnLB9eYnz2mR10S2BaHiEiU82tV27afecFiia40X5jHCAAAoB4RI6SSl3mNyv3uZ9t+2dxXtW3FOZoFf8KYldpWGbzsI8j3ZLUIcxzV5HLmZmtbWUPFCACANKErzRcqRkilKEa2BMHNvqIcbYfo+Tmvfuc7slUOp/uI8trmfZN+VIwAAEgRRqX5Q8UIcCjuc6DEaZZthM/LnF2Vnut1/q0or6+0RIoRLSpGAACkCTlGvlAxQiqVm13Xyzb8traBMHlZv6waLyMWbUVa/LweG3OUJfH9TVeaP8xjBAAAkKSI0XvvvSfXXXedzJw5U1auXCmdO3eW7373u3LFFVdI8+bNoy4eYsxPay+JLcWgo2iIv8LzWi5CEqe1w7zMiO01mhuH1xsKutLSXzFatGiR1NXVyeTJk2WvvfaSN954Q0aMGCEbNmyQm266KeriAQAQG3Sl+VOTyyVzWssJEybIpEmT5J133nH8nLVr10r79u1liJwo29U0C7R8iFa53ILC+5IgyNl/U9dKRqrE5fq0UY6tuS0ySx6TNWvWSLt27SQo+e+4fqf+TJo2397KNms3b5QFD10ReNnjJBERo1L0JHXo0CHqYgAAEC90pWWvYrRkyRK54447qnajbdq0ydwKa9PIhkq5BXGIrNgYLVPM7fpWUbfEsySpEUubquU75bm5Pp3mzXnJr4tLxMqrLHaBpWJU2mWXXSY1NTUVb5pfVOiDDz6QY445Rk455RSTZ1TJuHHjTFgxf+vWrVvArwgAACRZpDlGH374ofznP/+p+Jg999yzYeTZ8uXLZciQITJgwACZOnWqNGnSxHXESCtH5BhlTxAt9ji0KIlExFscrpE0cjrazuuoPK+PjU2O0SnXy3bN7OQYbd2yURY8fCU5RmHZeeedzc0JjRQdfvjh0q9fP7nvvvuqVopUixYtzA0AACA1OUZaKdJIUffu3U1ekUaa8jp16hRp2QAAiBOG62egYvTss8+ahGu9de3atdHfEjrbAELmJgyepKVAgnhdiMf5Scv5CnKgQbkBBV4mjay2r0RhVFr6lwQ588wzTQWo1A0AACBTESPEU1patMWCeD1ej5XNY+x2OD/C5XT5ChvCXOA1iteVdTV1n99sbStrqBgBAJAmdKX5QsUInhFZcN4K93qswjzGacttSaswJgONk1JRILc5RUDqcowAAIC7UWm2bm7MmTNHvvrVr0rnzp3NJM2PPvpo1efMmjVL+vbta6bX0YXidZ7CKBExiiFaP+Ecs6COc9InXUxKmaudP6d/L/WYMN+Dbvdlo0y2Xl8Yx8fJPogcFdGBSbYGJ+XcbWfDhg3Sp08fOfvss+XrX/961ce/++67cvzxx8t5550nDzzwgMyYMUPOPfdc2XXXXWXYsGESBSpGAADAimOPPdbcnLr77rtljz32kJtvvtn8vu+++8oLL7wgt956KxUj/FfcWjtxaoWVK4uXstl+PXE4Pmk//4X8zjNV6e9eo1BehLmvavtMGhZGTv4Ejy+++KIMHTq00X1aIRo1apREhYoRAACoug5bEEturVy5Ujp27NjoPv1d9/fZZ59Jy5YtJWxUjFCV7fwLP8+nRRg+m8fcb15Q1KKcCyruxyYKHIvwhut369at0d1jxoyRsWPHShpRMQIAIEWC6EpbtmyZtGvXruF+Wwu063qnq1atanSf/q77iiJapKgYxVhcW59hjmShlRwdP8feay6Yk30Vz2sTh2sjjNyjOLxOZFe7du0aVYxsGThwoDz55JPbrI+q90eFeYwAAEjjcH1bNxfWr18vr732mrnlh+Prv5cuXWp+Hz16tAwfPrzh8TpM/5133pFLLrlEFi1aJL/4xS/k97//vfzoRz+SqBAxirFKLUS/LVSbeUOMnoHfeWUqPS4JI4+cvj4nj7UpjtE1pHtU2vz58+Xwww9v+P3iiy82P8844wwzceOKFSsaKklKh+r/6U9/MhWh2267Tbp27Sr33ntvZEP1FRUjAABgxZAhQyRXIcpUalZrfc6rr74qcUHFKKFsRXxsrrtUvC3yg5LNxnmzMZtznK71ctuodu1H9R7gvZdRLCLrCzlGAAAA9YgYxZjNiIvNGaPLYRRNOpVa3dztObaZ6+I1X8lG/ly5baTl2ifKmw5Jmvk6jqgYAQCQJnW5z2+2tpUxVIwCEMXK1Tbmh4nz6DREp9JIMT/bCOr6sjE7td98OT8RMqd5TEHgvQtQMQIAIF1IvvaFipFLTlptbuY0cfJ4N+VyWhYveRi2W7JEmOCGreskjOut3HvEyf6d5jHFbc4kxEeNxdygGskeRqUBAADUq8lVmokpZdauXSvt27eXIXKibFfTzMo249I6i2P0JY5lQvYEcR1ybcONrbktMksekzVr1gSy3ljxd9yhR46V7bbb3so2t27dKH+dMTbwsscJXWkAAKQIw/X9oWLkU5Cjt9y0SuPYcnWb34Tke3viAPOzx6iXAttHHK4fosNAelExAgAgTRiV5gsVowgUz41SfH+538NouQcx23aem3WjaBUnQ/F58hsp8jLqs9pzvFxvXuctsjFfUZzmhAKyiIoRAAApUpPLmZutbWUNo9IsCiJ3KGz5SFPn2f+9LGyVsziKFefjAG/8Rl6SJu2vD8kclfaVw8ZYHZX2lznXZGpUGvMYAQAA1KMrLYDcB7+5Q2G0Qsvto1SOiK3yFG+bVnb6OL3W4xZpcVueMEbfAV7RleYPFSMAANKEUWm+UDFyyE3OhN9WcBj5GdVa8k4e61XcogXwzu/IKhvXgo1tuH2ujUhREGsmAvCPihEAAGmi3V+2usBy2QsZUTEKoVVqe1X6IPfpZs4XN8/x83jEQ6l1AYOc5yeKbfrdRxCz1RNhhVssCeIPo9IAAADqMY9RALxGVOLUMiwVHQCCunYrPT/K90Wc3pNIrrDnMRo88Eqr8xjNfvF65jECAADIInKMAuC1dRmnVmmYLff8nDCKeWGSydZITC/bDnJ0V5AjRIlGISg1dZ/fbG0ra6gYAQCQJoxK84WKkc81n7zk4sRl7ha3gtpXkLNtI/1rqYVxjfidrd7L/GBZW3cOiAsqRgAApAkzX/vCqDSHGBkDJEO1SEvUEdsgR/AhnsIelXZ4/8utjkp7fv4NjEoDAADIIiJGASiVTxB0CzYMcZxpGMnG+eYYZEHoEaN+o+1GjBaMI2IEAACQRSRfB6Bay6/470ltKfqdP6ZUSzmpxwLezrmTUVxeI6t+1iisdm17LUup51Q7Bl7/jgzTfiBb8w/lJHOoGAEAkCI1uZy52dpW1pBj5HO2ZiczNbuNrEQ5aoZWKGxym0+Xlusvrmu/IRs5RkcceJls19RSjlHtRpn56vhM5RgRMQIAIHXzGNma+Voyh4qRR27W9HLbMrTRkowiUkRLGMXc5gfZmBcoyLXTnHKzzzBzp5ARLAniC6PSAAAA6hExigGnLT0v67L53acbfmfytV0exIefvLpq12qYozzdvldLPc5veZ0eB2SYjkirsbitjKFiBABAijAqzR8qRhaVy21w2uKtNoInyOhOFDkRXvaNZHB7jZTLl3GzjTBEsd4auUNImrvuuksmTJggK1eulD59+sgdd9whBx98cMnHTp06Vc4666xG97Vo0UI2btwoUSHHCACANCZf27q58NBDD8nFF18sY8aMkVdeecVUjIYNGyarV68u+xydBmDFihUNt/fff1+iRMTIYqvM7yiROMyIXarM5SJXQY40QjaUu4aCuGZszC1k85r3+rkAxNktt9wiI0aMaIgC3X333fKnP/1JpkyZIpdddlnJ59TU1EinTp0kLogYAQCQJgFEjNauXdvotmnTpm12u3nzZlmwYIEMHTq04b4mTZqY31988cWyxV2/fr10795dunXrJieeeKK8+eabEiUiRgG0yuKQE+B1rhMno2iK72deFcQpGuIlCkX0BqkSwDxG3bp1a3S3dpWNHTu20X0fffSR1NbWSseOHRvdr78vWrSo5OZ79uxpokm9e/c2s2vfdNNNMmjQIFM56tq1q0SBihEAAKho2bJljZYE0QRpGwYOHGhueVop2nfffWXy5Mly3XXXSRSoGMU4T8GPMEeAMdoMQY3sTOp15Oc9TqQVcZzHqF27dlXXSttpp52kadOmsmrVqkb36+9Oc4iaNWsmBx54oCxZskSiQo4RAAApnMfI1s2p5s2bS79+/WTGjBkN99XV1ZnfC6NClWhX3MKFC2XXXXeVqBAxcikueQpuVym32Qq1tS1axunjN2pa6XlO10CLw/Vkc3QrkCQXX3yxnHHGGdK/f38zd9HEiRNlw4YNDaPUhg8fLl26dJFx48aZ36+99loZMGCA7LXXXvLJJ5+Y+Y90uP65554b2WugYgQAQJpEuIjsqaeeKh9++KFcffXVZoLHAw44QJ5++umGhOylS5eakWp5H3/8sRner4/dYYcdTMRp7ty5st9++0lUanK57Mz3rUMM27dvL0PkRNmuppnEVRDRHqetbRs5VLae5/e5CF4U56dUHlLYZShXFidliFMuIsKxNbdFZsljZtRVtTwdG99xQ3uMku2a2kmO3lq7SZ57e2LgZY8TcowAAADqETFC6GgJp4+taKKXiGUQ+XN5XKNIZMRoz4vsRozeuY2IEQAAQBaRfJ2SaIfXFe/djAZyu+1yUYC4HTs4U2nFe6/n1Mbs1F7y5Qr/XqkcUczq7nbEKbAti8nXkplOpQZUjAAASJMIR6WlATlGCZ+pN46txyBavOR+xFOS1smLY5mQDaHnGO1xoWzXxFKOUd0mee7dOzKVY0TECACANKnTeEfO4rayhYqRTzbmFvITWSmXC+G3dVwp/6Jc+Zzu00uZaOVHr9Q14XX2aaI3QIBydZ/fbG0rYxiVBgAAkOWI0Wdf7S9tn3jd3zZctHi9to7dPN7WKCE3zw9iHTbEl5ecsDBnSXd7vZb6m202Zn3P4/0Fx0i+9oWIEQAAQFIjRps2bZJDDjlEXn/9dXn11VfNAnVutXx8vojDUWlBrgIepLcnDjA/e4x6KbAcI6d/R7LYGDVYbltBznHl9H0X9fXq9vhGXV4kEMnX2YoYXXLJJdK5c+eoiwEAQLy70mzdMiZREaOnnnpK/vznP8sf/vAH8+8g2MhxKDenS/Hfg2wJeo0UBTETts11sRA8G6MGnUZabc5/FPUIUVtsjzAFkNKK0apVq2TEiBHy6KOPSqtWrRx3u+mtcPIrAABSzfSk2Uq+lsxJRMVIJ+c+88wz5bzzzpP+/fvLe++95+h548aNk2uuucbVvmyP7opLDpKNffptudpYFwvxUO5ayOe25fWY7m97YXMa3XUa1Sk1Es5tzl7UxwQJxKi05OYYXXbZZVJTU1PxtmjRIrnjjjtk3bp1Mnr0aFfb18frNOb527JlywJ7LQAAIPkiXSvtww8/lP/85z8VH7PnnnvKN7/5TXn88cdNRSmvtrZWmjZtKt/5znfk17/+dWRrpQU5gsxPlMdmC9zttpi7Jf3Knac45ep42WeY1zqyI/S10nY5V7Zr0tzKNrfWbZbnVt/LWmlh2Xnnnc2tmttvv12uv/76ht+XL18uw4YNk4ceesgM3QcAAPXoSkt/jtFuu+3W6Pc2bdqYnz169JCuXbsGuu8wIi/5SFFx5Ki4Fe4mT8FrWWyse+VnbTVa3sngNqJS/Lz8/csH11QdRek3iuhm5JvXbTOSDEiPRFSMAACAQ0SMkptjFDabOUZhtAjzEaTOs3OhzfdjY3tOW/h+1qwiFymZgnjfkBeEuAs9x2ins+3mGH00hRwjAACQUCwJ4gsVIwAAUiSXqzM3W9vKGipGHtns1irXLeRmWY8wu/bKlSuIxTHL7ZNukGSplpzc+aXPQ/TLB7ifnd7tsiJOJl10kxwe9hQBAIJFxQgAgDTR1GFbXWC57HWlkXwdg9ZcUvYRZcuWVnV22Vq02OZz4j5RJbKdfH1k+9NluxpLyde5zTJjzW8zlXwd6ZIgAAAAcZLJrrTPvtpf2j7xuqPHBrmIbH7bDXkM091vo1hxTkS5Vqbb+8vdFxZayeleKqT4+VGdb5v5cbbxHoBjdXUiNZaSpnPZS74mYgQAAFCPHKOUC3Opgqhb+4iPICJEtq+v/IhHL6PObPD7eni/JUfoOUZtvm03x2j9tEzlGGWyKw0AgLTK1dVJzlJXWi6DXWlEjALEKC4gPteyk+14yb0D4hYxOqLVt6xGjGZ++iARIwAAkFAm3sE8Rl5RMXLJTcsxykUto2zZel3U081zkB225i9ykmdXbbZsIBF0cscaKkZeMSoNAACgHhEjl4IY3RXGjNdh7MvrPmiNwwmn7zm/fwcSz0R5bM1jlJOsoWIEAECK5OpykrPUlZajYgSnvLQ6bUaZ3M4wTCsZSRfGqLQgngsgWcgxAgAgTXTuIZs3l+666y7ZfffdZfvtt5dDDjlEXn755YqPf/jhh2WfffYxj//Sl74kTz75pESJiFGIwhh1RqQIcRDHCIufssTpdQBx9tBDD8nFF18sd999t6kUTZw4UYYNGyaLFy+WXXbZZZvHz507V0477TQZN26cnHDCCTJt2jQ56aST5JVXXpH9998/ktdAxAgAgLTlGFm8uXHLLbfIiBEj5KyzzpL99tvPVJBatWolU6ZMKfn42267TY455hj56U9/Kvvuu69cd9110rdvX7nzzjslKlSMPLSEy61sD+C/ERaiLEC2utI2b94sCxYskKFDhzbc16RJE/P7iy++WPI5en/h45VGmMo9PgyZ6krLZ9dvlS2eJwXdumXj5z9zW2wWDQCQUuY7J8QRXn6+48qVXZcbKdSiRQtzK/TRRx9JbW2tdOzYsdH9+vuiRYuklJUrV5Z8vN4flUxVjNatW2d+viA+Ersef8xegQAAmfoO0rXMgtK8eXPp1KmTvLDSbvJymzZtpFu3bo3uGzNmjIwdO1bSKFMVo86dO8uyZcukbdu2UlNTE2lZtPatF5qWJysL84WFYxscjm2wOL7pPLYaKdJKkX4HBUlHdb377rumS8t2+WuKvjOLo0Vqp512kqZNm8qqVasa3a+/a4WtFL3fzePDkKmKkfZ1du3aVeJE36B8AAaDYxscjm2wOL7pO7ZBRoqKK0d6i0Lz5s2lX79+MmPGDDOyTNXV1ZnfL7jggpLPGThwoPn7qFGjGu579tlnzf1RyVTFCAAABOfiiy+WM844Q/r37y8HH3ywGa6/YcMGM0pNDR8+XLp06WKG56uLLrpIBg8eLDfffLMcf/zx8uCDD8r8+fPlnnvuiew1UDECAABWnHrqqfLhhx/K1VdfbRKoDzjgAHn66acbEqyXLl1qem/yBg0aZOYuuvLKK+Xyyy+XvffeWx599NHI5jBSVIwiov2zmrxWqp8W/nBsg8OxDRbHNzgc2/BccMEFZbvOZs2atc19p5xyirnFRU0uiyvEAQAAlMAEjwAAAPWoGAEAANSjYgQAAFCPilGMbNq0yWTw60Rar732WtTFSbz33ntPzjnnHNljjz2kZcuW0qNHD5N8aXvysyy56667ZPfddzfzpOjK2S+//HLURUo8HbZ80EEHmYlndfVxnf9FVyKHfePHjzefr4Vz5gDFqBjFyCWXXBL4zKhZomvz6ORikydPljfffFNuvfVWs9KzDgmFew899JCZo0Qrl6+88or06dPHLPa4evXqqIuWaLNnz5aRI0fKSy+9ZCa227Jlixx99NFm7hfYM2/ePPNZ0Lt376iLgphjVFpMPPXUU+ZL5w9/+IP06tVLXn31VRM9gl0TJkyQSZMmyTvvvBN1URJHI0Qa2bjzzjvN71rp1CUWLrzwQrnsssuiLl5q6BwwGjnSCtNhhx0WdXFSYf369dK3b1/5xS9+Iddff735bNWJB4FSiBjFgK4LM2LECPntb38rrVq1iro4qbZmzRrp0KFD1MVIHO1+XLBggQwdOrThPp2kTX9/8cUXIy1bGq9RxXVqj0bkdFblwusXKIcJHiOmAbszzzxTzjvvPDOFuubFIBhLliyRO+64Q2666aaoi5I4H330kdTW1jbMXpunv2uXJezQKJzmvxx66KGRzvybJrrEhHb9alca4AQRo4Bo14Im+VW66ReKflHrqsujR4+OusipO7aFPvjgAznmmGPM7KoanQPiGtl44403zJc5/Fu2bJlZi+uBBx6IbGFVJA85RgHmCfznP/+p+Jg999xTvvnNb8rjjz9uvszztGXetGlT+c53viO//vWvQyhtOo+trvSsli9fLkOGDJEBAwbI1KlTG63TA+ddadrN+8gjjzSsmq10schPPvlEHnvssUjLlwa6hIIexzlz5piRlPBP19w6+eSTzedp4eerft7q54COBC78G6CoGEVMF9Rbu3Ztw+/6Ja4jffQLSJNdu3btGmn5kk4jRYcffrj069dP7r//fj4EfdDrUVfL1ihnvttnt912M1/oJF97px/BmsA+ffp0s46ULqIJOzQa//777ze6T1d532effeTSSy+luxIlkWMUMf1iKdSmTRvzU+fcoVLkv1KkkaLu3bubvCKNNOV16tQp0rIlkY6a1AiR5sJpBUlH9eiQcv2igb/uM11dXKNFOpeRrkiu2rdvb+bfgnd6PIsrP61bt5Ydd9yRShHKomKE1NI5YTThWm/FlUwCpe6deuqppnJ59dVXmy9vHfL89NNPb5OQDXd0+gillfhC9913nxmYASBcdKUBAADUIwsVAACgHhUjAACAelSMAAAA6lExAgAAqEfFCAAAoB4VIwAAgHpUjAAAAOpRMQIAAKhHxQgAAKAeFSMAAIB6VIwAAADqUTEC0IguFNupUye54YYbGu6bO3euNG/eXGbMmBFp2QAgaFSMADSy8847y5QpU2Ts2LEyf/58WbdunZx++ulywQUXyJFHHikXXnihdOjQYZvV4AEgDWpyuVwu6kIAiJ+RI0fKc889J/3795eFCxfKvHnzpEWLFvLGG2/I0qVL5ec//7nMmjUr6mICgFVEjACUdNNNN8nWrVvl4YcflgceeMBUitT+++8vrVq1irp4ABAIKkYASnr77bdl+fLlUldXJ++9917UxQGAUGwXzm4AJMnmzZvlu9/9rpx66qnSs2dPOffcc0132i677BJ10QAgUESMAGzjiiuukDVr1sjtt98ul156qXzxi1+Us88+O+piAUDgSL4G0IgmVB911FHy/PPPy5e//GVzn3al9enTR8aPH2/+ff/998v//d//SZcuXWTmzJmy2267RV1sALCCihEAAEA9utIAAADqUTECAACoR8UIAACgHhUjAACAelSMAAAA6lExAgAAqEfFCAAAoB4VIwAAgHpUjAAAAOpRMQIAAKhHxQgAAKAeFSMAAAD53P8D0K806+xF6vkAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# ---- Discrete modality -------------------------------------------------\n", + "discrete_samples = samples[0].cpu().numpy() # shape (N, 2) integer tokens\n", + "vocab = vocab_size\n", + "\n", + "# Plot a 2‑D histogram of the discrete samples\n", + "plt.figure(figsize=(6, 5))\n", + "plt.hist2d(\n", + " discrete_samples[:, 0],\n", + " discrete_samples[:, 1],\n", + " bins=vocab,\n", + " cmap=\"viridis\",\n", + ")\n", + "plt.title(\"Discrete modality samples (token histogram)\")\n", + "plt.xlabel(\"Token 1\")\n", + "plt.ylabel(\"Token 2\")\n", + "plt.colorbar(label=\"Count\")\n", + "plt.tight_layout()\n", + "plt.show()\n", + "\n", + "# ---- Continuous modality -----------------------------------------------\n", + "continuous_samples = samples[1].cpu().numpy() # shape (N, 2)\n", + "\n", + "# Plot a 2‑D histogram of the continuous samples\n", + "plt.figure(figsize=(6, 5))\n", + "plt.hist2d(\n", + " continuous_samples[:, 0],\n", + " continuous_samples[:, 1],\n", + " bins=200,\n", + " cmap=\"viridis\",\n", + ")\n", + "plt.title(\"Continuous modality samples (2-D density)\")\n", + "plt.xlabel(\"x₁\")\n", + "plt.ylabel(\"x₂\")\n", + "plt.colorbar(label=\"Count\")\n", + "plt.tight_layout()\n", + "plt.show()" + ] + } + ], + "metadata": { + "jupytext": { + "cell_metadata_filter": "-all", + "main_language": "python", + "notebook_metadata_filter": "-all" + }, + "kernelspec": { + "display_name": "flow_matching", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/README.md b/examples/README.md index 3b6af05..45542bb 100644 --- a/examples/README.md +++ b/examples/README.md @@ -16,6 +16,6 @@ | [standalone_discrete_flow_matching.ipynb](standalone_discrete_flow_matching.ipynb) | A concise discrete flow matching example built in pure PyTorch. | | [2d_flow_matching.ipynb](2d_flow_matching.ipynb) | 2D flow matching example on the checkerboard dataset using the flow_matching library. | | [2d_discrete_flow_matching.ipynb](2d_discrete_flow_matching.ipynb) | 2D discrete flow matching example on the checkerboard dataset using the flow_matching library. | +| [2d_multimodal_flow_matching.ipynb](2d_multimodal_flow_matching.ipynb) | 2D multimodal (discrete-continuous) flow matching on the checkerboard dataset and the flow_matching library. | | [2d_riemannian_flow_matching_flat_torus.ipynb](2d_riemannian_flow_matching_flat_torus.ipynb) | 2D Riemannian flow matching on a flat torus on the checkerboard dataset and the flow_matching library. | | [2d_riemannian_flow_matching_sphere.ipynb](2d_riemannian_flow_matching_sphere.ipynb) | 2D Riemannian flow matching on a sphere on the checkerboard dataset and the flow_matching library. | - diff --git a/flow_matching/solver/__init__.py b/flow_matching/solver/__init__.py index 6bd7b01..3a3bedb 100644 --- a/flow_matching/solver/__init__.py +++ b/flow_matching/solver/__init__.py @@ -14,5 +14,6 @@ "Solver", "ModelWrapper", "MixtureDiscreteEulerSolver", + "MultimodalSolver", "RiemannianODESolver", ] diff --git a/flow_matching/solver/multimodal_solver.py b/flow_matching/solver/multimodal_solver.py new file mode 100644 index 0000000..029e1a6 --- /dev/null +++ b/flow_matching/solver/multimodal_solver.py @@ -0,0 +1,355 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the CC-by-NC license found in the +# LICENSE file in the root directory of this source tree. + +from contextlib import nullcontext +from math import ceil +from typing import Any, Callable, Dict, List, Optional, Sequence, Union + +import torch +from torch import Tensor + +from torch.nn import functional as F + +from flow_matching.path import MixtureDiscreteProbPath +from flow_matching.solver.solver import Solver +from flow_matching.solver.utils import get_nearest_times +from flow_matching.utils import categorical, expand_tensor_like, ModelWrapper + +try: + from tqdm import tqdm + + TQDM_AVAILABLE = True +except ImportError: + TQDM_AVAILABLE = False + + +class MultimodalSolver(Solver): + """Solver for multiple continuous and discrete data modalities. + + This solver handles an arbitrary number of modalities, which can be either + continuous or discrete. Each modality has its own state tensor. + All modalities share the same time discretization and are updated + simultaneously at each step. + + For continuous modalities, an Euler integration step is used. For discrete + modalities, the update follows the procedure from `MixtureDiscreteEulerSolver`. + + Args: + model (Union[ModelWrapper, Callable]): + A model that receives a sequence of state tensors + (one per modality) as ``x`` and a scalar time tensor ``t``, + and returns a sequence of output tensors. For continuous modalities, + the output is a velocity. For discrete modalities, it is the + posterior probability `p_1t`. + modality_configs (List[Dict[str, Any]]): + A list of configuration dictionaries, one for each modality. + Each dictionary must have a ``'type'`` key, which is either + ``'continuous'`` or ``'discrete'``. Discrete modality configs may + provide a ``'dtype_categorical'`` key with the desired data type + for categorical logit sampling (e.g., ``torch.float32``) and + must provide a ``'path'`` key with a `MixtureDiscreteProbPath` + instance. Continuous modality configs must provide a ``'path'`` + key with a `ProbPath` instance + (e.g., `AffineProbPath(scheduler=CondOTScheduler())`) as well as + an ``'x_1_prediction'`` key which is either ``True`` or ``False``. + If ``True``, the model is expected to predict the clean data `x_1` + for that modality, and such predictions will be reparameterized + as velocities during the sampling process. If ``False``, the model + is expected to predict the velocities directly. + source_distribution_p (Optional[Tensor], optional): Source distribution, + must be of shape [vocabulary_size]. Required only when divergence-free + term for the probability velocity is non-zero. Defaults to None. + model_sampling_fn (str, optional): If ``model`` is a class instance + with multiple methods, this specifies the method to use for + forward passes during sampling. Defaults to ``"forward"``. + + Raises: + TypeError: If ``model`` is not callable or if ``modality_configs`` + is not a list of dictionaries. + """ + + def __init__( + self, + model: Union[ModelWrapper, Callable], + modality_configs: List[Dict[str, Any]], + source_distribution_p: Optional[Tensor] = None, + model_sampling_fn: str = "forward", + ): + super().__init__() + if not callable(model): + raise TypeError(f"model must be callable, got {type(model)}") + self.model = model + self.modality_configs = modality_configs + self.source_distribution_p = source_distribution_p + self.model_sampling_fn = model_sampling_fn + + self._validate_configs() + + def _validate_configs(self): + """Validates the modality configurations.""" + if not isinstance(self.modality_configs, list): + raise TypeError("modality_configs must be a list of dictionaries.") + for i, config in enumerate(self.modality_configs): + if not isinstance(config, dict): + raise TypeError(f"Config for modality {i} must be a dictionary.") + if "type" not in config: + raise ValueError(f"Config for modality {i} must have a 'type' key.") + if config["type"] not in ["continuous", "discrete"]: + raise ValueError( + f"Unsupported modality type '{config['type']}' for modality {i}." + ) + if config["type"] == "discrete": + if "path" not in config: + raise ValueError( + f"Discrete modality {i} requires a 'path' in its config." + ) + if not isinstance(config["path"], MixtureDiscreteProbPath): + raise TypeError( + f"'path' for discrete modality {i} must be a MixtureDiscreteProbPath instance." + ) + if config["type"] == "continuous": + if "path" not in config: + raise ValueError( + f"Continuous modality {i} requires a 'path' in its config." + ) + if "x_1_prediction" not in config: + raise ValueError( + f"Continuous modality {i} requires an 'x_1_prediction' key in its config." + ) + if not isinstance(config["x_1_prediction"], bool): + raise TypeError( + f"'x_1_prediction' for continuous modality {i} must be a boolean." + ) + + def sample( + self, + x_init: Sequence[Tensor], + step_size: Optional[float], + div_free: Union[float, Callable[[float], float]] = 0.0, + method: str = "euler", + time_grid: Tensor = torch.tensor([0.0, 1.0]), + return_intermediates: bool = False, + enable_grad: bool = False, + verbose: bool = False, + **model_extras: dict, + ) -> Union[Sequence[Tensor], Sequence[List[Tensor]]]: + """Sample all modalities simultaneously. + + Args: + x_init (Sequence[Tensor]): Initial states for each modality. + step_size (Optional[float]): Fixed step size for uniform discretization. + If ``None``, the discretization is taken from ``time_grid``. + div_free (Union[float, Callable[[float], float]]): The coefficient + of the divergence-free term in the probability velocity + (for discrete modalities). Can be either a float or a time + dependent function. Defaults to 0.0. + method (str): Numerical integration method. Currently only ``"euler"`` is + supported, representing a single forward step. + time_grid (Tensor): Tensor of time points defining the interval. + return_intermediates (bool): If ``True``, returns a list of tensors for + each modality containing the state at each intermediate time step. + enable_grad (bool): Whether to enable gradient tracking during integration. + verbose (bool): If ``True``, displays a progress bar during sampling. + **model_extras (dict): Additional arguments passed to the model. + + Raises: + ValueError: If the number of initial states does not match the number of + modality configurations. + NotImplementedError: If an unsupported integration method is specified. + ImportError: If ``verbose`` is ``True`` but ``tqdm`` is not installed. + TypeError: If the model's output does not match the expected format. + + Returns: + Union[Sequence[Tensor], Sequence[List[Tensor]]]: If ``return_intermediates`` is + ``False`` (default), returns a list of final state tensors, one per + modality. If ``True``, returns a list where each element is another + list of tensors representing the trajectory for a modality. + """ + if len(x_init) != len(self.modality_configs): + raise ValueError( + "Number of initial states must match the number of modality configurations." + ) + if method != "euler": + raise NotImplementedError( + f"Method '{method}' is not implemented for MultimodalSolver." + ) + if not div_free == 0.0: + assert ( + self.source_distribution_p is not None + ), "Source distribution p must be specified in order to add a divergence-free term to the probability velocity for each discrete modality." + + # Initialize the current state `x_t` with the initial state `X_0`. + device = x_init[0].device + batch_size = x_init[0].shape[0] + time_grid = time_grid.to(device) + + if step_size is None: + # If step_size is None then set the t discretization to time_grid. + t_discretization = time_grid + n_steps = len(time_grid) - 1 + else: + # If step_size is float then t discretization is uniform with step size set by step_size. + t_init = time_grid[0].item() + t_final = time_grid[-1].item() + assert ( + t_final - t_init + ) > step_size, f"Time interval [time_grid[0], time_grid[-1]] must be larger than step_size. Got a time interval [{t_init}, {t_final}] and step_size {step_size}." + + n_steps = ceil((t_final - t_init) / step_size) + t_discretization = torch.tensor( + [t_init + step_size * i for i in range(n_steps)] + [t_final], + device=device, + ) + + if return_intermediates: + # Get order of intermediate steps + order = torch.argsort(time_grid) + # Compute intermediate steps to return via nearest points in t_discretization to time_grid. + time_grid = get_nearest_times( + time_grid=time_grid, t_discretization=t_discretization + ) + + states: Sequence[Tensor] = [(x if enable_grad else x.clone()) for x in x_init] + intermediates: Sequence[List[Tensor]] = ( + [[x if enable_grad else x.clone()] for x in x_init] + if return_intermediates + else [] + ) + + steps_counter = 0 + + if verbose: + if not TQDM_AVAILABLE: + raise ImportError( + "tqdm is required for verbose mode. Please install it." + ) + ctx = tqdm(total=t_final, desc=f"NFE: {steps_counter}") + else: + ctx = nullcontext() + + with ctx, torch.set_grad_enabled(enable_grad): + for i in range(n_steps): + # NOTE: For now, all modalities share the same time + t = [t_discretization[i : i + 1].repeat(batch_size)] * len(states) + h = t_discretization[i + 1 : i + 2] - t_discretization[i : i + 1] + + model_fn = getattr(self.model, self.model_sampling_fn, self.model) + outputs = model_fn(states, t, **model_extras) + + if not isinstance(outputs, (list, tuple)) or len(outputs) != len( + states + ): + raise TypeError( + "The model must return a sequence of tensors matching the number of modalities." + ) + + for idx, config in enumerate(self.modality_configs): + model_output = outputs[idx] + + t_expanded = expand_tensor_like( + input_tensor=t[idx], + expand_to=model_output, + ) + + if config["type"] == "continuous": + # Sample x_{t+h} = x_t + h * v(x_t,t) + path = config["path"] + velocity_output = ( + path.target_to_velocity( + x_1=model_output, x_t=states[idx], t=t_expanded + ) + if config["x_1_prediction"] + else model_output + ) + + states[idx] = states[idx] + h * velocity_output + + elif config["type"] == "discrete": + dtype = config.get("dtype_categorical", torch.float32) + + # Sample x_1 ~ p_1|t( \cdot |x_t) + p_1t = torch.softmax(model_output, dim=-1) + x_1 = categorical(p_1t.to(dtype=dtype)) + + # Checks if final step + if i == n_steps - 1: + states[idx] = x_1 # x_t = x_1 at final step + else: + vocabulary_size = p_1t.shape[-1] + if self.source_distribution_p is not None: + assert self.source_distribution_p.shape == torch.Size( + [vocabulary_size] + ), f"Source distribution p dimension must match the vocabulary size {vocabulary_size}. Got {self.source_distribution_p.shape}." + + # Compute u_t(x|x_t,x_1) + path: MixtureDiscreteProbPath = config["path"] + scheduler_output = path.scheduler(t=t_expanded) + + k_t = scheduler_output.alpha_t + d_k_t = scheduler_output.d_alpha_t + + delta_1 = F.one_hot(x_1, num_classes=vocabulary_size).to( + k_t.dtype + ) + u = d_k_t / (1 - k_t) * delta_1 + + # Add divergence-free part + div_free_t = ( + div_free(t_expanded) if callable(div_free) else div_free + ) + + if div_free_t > 0: + p_0 = self.source_distribution_p[ + (None,) * states[idx].dim() + ] + u = u + div_free_t * d_k_t / (k_t * (1 - k_t)) * ( + (1 - k_t) * p_0 + k_t * delta_1 + ) + + # Set u_t(x_t|x_t,x_1) = 0 + delta_t = F.one_hot( + states[idx], num_classes=vocabulary_size + ) + u = torch.where( + delta_t.to(dtype=torch.bool), torch.zeros_like(u), u + ) + + # Sample x_t ~ u_t( \cdot |x_t,x_1) + intensity = u.sum(dim=-1) # Assuming u_t(xt|xt,x1) := 0 + mask_jump = torch.rand( + size=states[idx].shape, device=states[idx].device + ) < 1 - torch.exp(-h * intensity) + + if mask_jump.sum() > 0: + states[idx][mask_jump] = categorical( + u[mask_jump].to(dtype=dtype) + ) + + # Increment time for each modality + t[idx] = t[idx] + h + + steps_counter += 1 + + if return_intermediates: + for idx, s in enumerate(states): + if t[idx] in time_grid: + intermediates[idx].append(s if enable_grad else s.clone()) + + if verbose: + ctx.n = (torch.cat(t) * n_steps).mean().long().item() + ctx.refresh() + ctx.set_description(f"NFE: {steps_counter}") + + if return_intermediates: + if step_size is None: + return intermediates + else: + return [ + [intermediates[idx][i] for i in order] + for idx in range(len(intermediates)) + ] + else: + return states diff --git a/flow_matching/utils/multimodal.py b/flow_matching/utils/multimodal.py new file mode 100644 index 0000000..f239aa8 --- /dev/null +++ b/flow_matching/utils/multimodal.py @@ -0,0 +1,287 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the CC-by-NC license found in the +# LICENSE file in the root directory of this source tree. + +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union + +import torch +from torch import nn, Tensor + +# flow_matching +from flow_matching.loss.generalized_loss import MixturePathGeneralizedKL +from flow_matching.path.mixture import MixtureDiscreteProbPath +from flow_matching.solver.multimodal_solver import MultimodalSolver + + +MULTIMODAL_METHOD = Literal["euler"] + + +def _default_continuous_loss( + pred: Tensor, target: Tensor, reduction: str = "none" +) -> Tensor: + """ + Squared error loss for continuous modalities. + + Args: + pred (Tensor): predicted velocity field. + target (Tensor): target velocity field. + reduction (str): reduction method, one of 'mean', 'sum', or 'none'. + + Raises: + ValueError: if reduction is not one of 'none', 'mean', or 'sum'. + + Returns: + Tensor: computed loss. + """ + loss = (pred - target) ** 2 + + if reduction == "mean": + return torch.mean(loss) + elif reduction == "sum": + return torch.sum(loss) + elif reduction == "none": + return loss + else: + raise ValueError("reduction must be one of 'none', 'mean', or 'sum'") + + +class Flow(nn.Module): + """ + Generic multimodal flow matching model. + + This class aggregates multiple modalities, each with its own model, path, + scheduler, and loss. It provides utilities for training (computing the total + loss) and inference (sampling) across all modalities. + + Args: + model (nn.Module): + A model that receives a sequence of state tensors + (one per modality) as ``x`` and a scalar time tensor ``t``, + and returns a sequence of output tensors. For continuous modalities, + the output is a velocity. For discrete modalities, it is the + posterior probability `p_1t`. + modalities (Dict[str, Dict[str, Any]]): + Mapping from modality name to a dict with keys: + - "path": A probability path object (e.g., MixtureDiscreteProbPath for discrete data, + or any continuous path implementation). + - "loss" (optional): A callable loss function. If omitted, a default loss is chosen + based on the path type. + - "weight" (optional): A float weight for the modality's training loss. Defaults to 1.0. + - "x_1_prediction" (continuous paths only, optional): If True, the model is expected to predict + the clean data `x_1` for that modality, and such predictions will be reparameterized + as velocities during the sampling process. If False, the model is expected to predict + the velocities directly. Defaults to False. + model_sampling_fn (str, optional): If ``model`` is a class instance + with multiple methods, this specifies the method to use for + forward passes during sampling. Defaults to ``"forward"``. + """ + + def __init__( + self, + model: nn.Module, + modalities: Dict[str, Dict[str, Any]], + model_sampling_fn: str = "forward", + ) -> None: + super().__init__() + self.model = model + self.paths: Dict[str, Any] = {} + self.loss_fns: Dict[str, Callable] = {} + self.loss_weights: Dict[str, float] = {} + + for name, spec in modalities.items(): + path = spec["path"] + self.paths[name] = path + + # Choose loss function + loss_fn = spec.get("loss") + if loss_fn is None: + if isinstance(path, MixtureDiscreteProbPath): + loss_fn = MixturePathGeneralizedKL(path, reduction="none") + else: + loss_fn = _default_continuous_loss + self.loss_fns[name] = loss_fn + self.loss_weights[name] = spec.get("weight", 1.0) + + # Set up Euler solver for each modality. + self.modality_configs = [ + { + "name": name, + "type": ( + "discrete" + if isinstance(path, MixtureDiscreteProbPath) + else "continuous" + ), + "path": path, + "x_1_prediction": modalities[name].get("x_1_prediction", False), + } + for name, path in self.paths.items() + ] + self.solver = MultimodalSolver( + model=self.model, + modality_configs=self.modality_configs, + model_sampling_fn=model_sampling_fn, + ) + + def training_loss( + self, + x_1: Sequence[Tensor], + x_t: Sequence[Tensor], + dx_t: Sequence[Tensor], + t: Sequence[Tensor], + model_output: Optional[Sequence[Tensor]] = None, + detach_loss_dict: bool = True, + **model_extras: dict, + ) -> Tuple[Sequence[Tensor], Dict[str, Tensor]]: + """ + Compute the total training loss across all modalities. + + Args: + x_1 (Sequence[Tensor]): Sequence of tensors, one per modality, + containing the data at time 1. + x_t (Sequence[Tensor]): Sequence of tensors, one per modality, + containing the data at time t. + dx_t (Sequence[Tensor]): Sequence of tensors, one per modality, + containing the velocity field at time t. + t (Sequence[Tensor]): Sequence of tensors, one per modality, + containing the time values. + model_output (Optional[Sequence[Tensor]]): Optional precomputed model outputs. + If provided, these are used instead of calling the model. + detach_loss_dict (bool): If ``True``, detaches individual modality losses + from the computation graph when storing them in the loss dictionary. + Defaults to ``True``. + **model_extras (dict): Additional keyword arguments to pass to the model. + + Returns: + Tuple[Sequence[Tensor], Dict[str, Tensor]]: + Scalar loss (sum of modality losses) and a dictionary + of individual modality losses. + """ + assert ( + len(x_1) == len(x_t) == len(dx_t) == len(t) == len(self.paths) + ), "Input sequences must match the number of modalities." + + if model_output is not None: + assert len(model_output) == len( + self.paths + ), "If provided, model outputs must match the number of modalities." + + loss_dict = {} + total_loss = 0.0 + + model_output = model_output or self.model(x_t, t, **model_extras) + + for i, name in enumerate(self.paths): + path = self.paths[name] + loss_fn = self.loss_fns[name] + modality_config = self.modality_configs[i] + + if isinstance(path, MixtureDiscreteProbPath): + # Discrete case: model should output logits. + assert x_t[i].dtype == torch.long, ( + f"Expected integer tensor for discrete modality '{name}', " + f"got {x_t[i].dtype}", + ) + loss = loss_fn(model_output[i], x_1[i], x_t[i], t[i]) + else: + # Continuous case: model returns velocity field. + assert x_t[i].is_floating_point(), ( + f"Expected float tensor for continuous modality '{name}', " + f"got {x_t[i].dtype}", + ) + loss = loss_fn( + model_output[i], + x_1[i] if modality_config["x_1_prediction"] else dx_t[i], + ) + + weight = self.loss_weights[name] + loss_dict[name] = (loss.detach() if detach_loss_dict else loss) * weight + total_loss = total_loss + loss.mean() * weight + + return total_loss, loss_dict + + def sample( + self, + x_init: Sequence[Tensor], + time_grid: Optional[Tensor] = None, + device: torch.device = torch.device("cpu"), + steps: int = 1000, + step_size: Optional[float] = None, + div_free: Union[float, Callable[[float], float]] = 0.0, + method: MULTIMODAL_METHOD = "euler", + return_intermediates: bool = False, + enable_grad: bool = False, + verbose: bool = False, + **model_extras: dict, + ) -> Union[Sequence[Tensor], Sequence[List[Tensor]]]: + """ + Generate samples for each modality using the inference scheduler. + + Args: + x_init (Sequence[Tensor]): + Sequence of tensors, one per modality, containing the initial states at time 0. + For continuous modalities, this is typically Gaussian noise. + For discrete modalities, this is typically samples from a uniform categorical distribution. + time_grid (Optional[Tensor]): Optional tensor of time points defining the interval. + If provided, it overrides the uniform discretization defined by `steps`. + device (torch.device, optional): Device on which to run the sampling. + steps (int, optional): Number of integration steps for the ODE solver. + step_size (Optional[float]): Fixed step size for uniform discretization. + If ``None``, the step size is computed from ``steps``. + div_free (Union[float, Callable[[float], float]]): The coefficient + of the divergence-free term in the probability velocity + (for discrete modalities). Can be either a float or a time + dependent function. Defaults to 0.0. + method (MULTIMODAL_METHOD): Numerical integration method. Currently only ``"euler"`` is + supported, representing a single forward step. + return_intermediates (bool): If ``True``, returns a list of tensors for + each modality containing the state at each intermediate time step. + enable_grad (bool): Whether to enable gradient tracking during integration. + verbose (bool): If ``True``, prints progress during sampling. + **model_extras (dict): Additional keyword arguments to pass to the model. + + Returns: + Union[Sequence[Tensor], Sequence[List[Tensor]]]: A list where each element corresponds to a modality. + Each element is either a tensor of shape ``(batch_size, ...)`` containing the samples, + or a list of tensors (if `return_intermediates` is True in `MultimodalSolver.sample`). + """ + # Validate samples for each modality. + x_init = x_init if isinstance(x_init, list) else list(x_init) + for i, name in enumerate(self.paths): + path = self.paths[name] + + if isinstance(path, MixtureDiscreteProbPath): + assert x_init[i].dtype == torch.long, ( + f"Expected integer tensor for discrete modality '{name}', " + f"got {x_init[i].dtype}", + ) + else: + assert x_init[i].is_floating_point(), ( + f"Expected float tensor for continuous modality '{name}', " + f"got {x_init[i].dtype}", + ) + + x_init[i] = x_init[i].to(device) + + # Solve to obtain multimodal samples at time 1. + step_size = step_size or (1.0 / steps) + time_grid = ( + time_grid + if time_grid is not None + else torch.linspace(0.0, 1.0, steps, device=device) + ) + + samples = self.solver.sample( + x_init=x_init, + step_size=step_size, + div_free=div_free, + method=method, + time_grid=time_grid, + return_intermediates=return_intermediates, + enable_grad=enable_grad, + verbose=verbose, + **model_extras, + ) + + return samples diff --git a/tests/solver/test_multimodal_solver.py b/tests/solver/test_multimodal_solver.py new file mode 100644 index 0000000..9a844af --- /dev/null +++ b/tests/solver/test_multimodal_solver.py @@ -0,0 +1,249 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the CC-by-NC license found in the +# LICENSE file in the root directory of this source tree. +import unittest +from unittest.mock import MagicMock + +import torch +from flow_matching.path import AffineProbPath, MixtureDiscreteProbPath +from flow_matching.path.scheduler import CondOTScheduler, PolynomialConvexScheduler + +from flow_matching.solver.multimodal_solver import MultimodalSolver +from flow_matching.utils import ModelWrapper +from torch import Tensor + + +# ---------------------------------------------------------------------- +# Helper models for continuous and discrete modalities +# ---------------------------------------------------------------------- +class ContinuousVelocityModel(ModelWrapper): + def __init__(self): + super().__init__(None) + + def forward(self, xs: list[Tensor], t: list[Tensor], **extras) -> list[Tensor]: + # xs is a list of modality states; we only have one continuous modality here. + # Return a list with the same length as xs. + return [2.0 * xs[0]] + + +class DiscreteLogitsModel(ModelWrapper): + def __init__(self, vocab_size: int): + super().__init__(None) + self.vocab_size = vocab_size + + def forward(self, xs: list[Tensor], t: list[Tensor], **extras) -> list[Tensor]: + """Produce logits that give probability 1.0 to the last class.""" + batch = xs[0].shape[0] + logits = torch.full((batch, self.vocab_size), -1e9, device=xs[0].device) + logits[:, -1] = 1e9 + return [logits] + + +# ---------------------------------------------------------------------- +# Test suite +# ---------------------------------------------------------------------- +class TestMultimodalSolver(unittest.TestCase): + def setUp(self): + # Continuous modality config (no extra args needed) + self.continuous_cfg = { + "type": "continuous", + "path": AffineProbPath(scheduler=CondOTScheduler()), + "x_1_prediction": False, + } + + # Discrete modality config + self.vocab_size = 3 + self.discrete_path = MixtureDiscreteProbPath( + scheduler=PolynomialConvexScheduler(n=2.0) + ) + self.discrete_cfg = { + "type": "discrete", + "path": self.discrete_path, + } + + # Source distribution for divergence‑free term (uniform) + self.source_p = torch.tensor([1.0 / self.vocab_size] * self.vocab_size) + + # Dummy models + self.continuous_model = ContinuousVelocityModel() + self.discrete_model = DiscreteLogitsModel(vocab_size=self.vocab_size) + + # Combined model that forwards to the appropriate sub‑model + class CombinedModel(ModelWrapper): + def __init__(self, cont, disc): + super().__init__(None) + self.cont = cont + self.disc = disc + + def forward(self, xs, t, **extras): + # xs[0] -> continuous, xs[1] -> discrete + cont_out = self.cont.forward([xs[0]], t, **extras)[0] + disc_out = self.disc.forward([xs[1]], t, **extras)[0] + return [cont_out, disc_out] + + self.model = CombinedModel(self.continuous_model, self.discrete_model) + + # ------------------------------------------------------------------ + # Basic initialization test + # ------------------------------------------------------------------ + def test_init(self): + solver = MultimodalSolver( + model=self.model, + modality_configs=[self.continuous_cfg, self.discrete_cfg], + source_distribution_p=self.source_p, + ) + self.assertIs(solver.model, self.model) + self.assertEqual( + solver.modality_configs, [self.continuous_cfg, self.discrete_cfg] + ) + self.assertTrue(torch.allclose(solver.source_distribution_p, self.source_p)) + + # ------------------------------------------------------------------ + # Simple sampling test (continuous + discrete) + # ------------------------------------------------------------------ + def test_sample_basic(self): + solver = MultimodalSolver( + model=self.model, + modality_configs=[self.continuous_cfg, self.discrete_cfg], + source_distribution_p=self.source_p, + ) + # Initial states: continuous (batch=1, dim=1), discrete (batch=1, categorical) + x_cont = torch.tensor([[0.0]]) # shape (1, 1) + x_disc = torch.tensor([[0]]) # shape (1, 1) + result = solver.sample( + x_init=[x_cont, x_disc], + step_size=0.1, + time_grid=torch.tensor([0.0, 1.0]), + ) + # Continuous modality: v = 2*x, Euler step => x_final = x0 + h*2*x0 = 0 + # Discrete modality: logits always select last class => final state = vocab_size-1 + self.assertTrue(torch.allclose(result[0], torch.zeros_like(result[0]))) + self.assertTrue(torch.equal(result[1], torch.tensor([self.vocab_size - 1]))) + + # ------------------------------------------------------------------ + # Return intermediates test + # ------------------------------------------------------------------ + def test_return_intermediates(self): + solver = MultimodalSolver( + model=self.model, + modality_configs=[self.continuous_cfg, self.discrete_cfg], + source_distribution_p=self.source_p, + ) + x_cont = torch.tensor([[1.0]]) # start at 1.0 + x_disc = torch.tensor([[0]]) # start at class 0 + intermediates = solver.sample( + x_init=[x_cont, x_disc], + step_size=0.5, + time_grid=torch.tensor([0.0, 0.5, 1.0]), + return_intermediates=True, + ) + # Should return a list of two lists (one per modality) + self.assertEqual(len(intermediates), 2) + # Continuous trajectory should have three entries (including start & end) + self.assertEqual(len(intermediates[0]), 3) + # Discrete trajectory should also have three entries + self.assertEqual(len(intermediates[1]), 3) + # Verify the final discrete state is the last class + self.assertTrue( + torch.equal(intermediates[1][-1], torch.tensor([self.vocab_size - 1])) + ) + + # ------------------------------------------------------------------ + # Gradient tracking test + # ------------------------------------------------------------------ + def test_gradient_enabled(self): + solver = MultimodalSolver( + model=self.model, + modality_configs=[self.continuous_cfg, self.discrete_cfg], + source_distribution_p=self.source_p, + ) + x_cont = torch.tensor([[2.0]], requires_grad=True) + x_disc = torch.tensor([[0]], requires_grad=False) + result = solver.sample( + x_init=[x_cont, x_disc], + step_size=0.1, + time_grid=torch.tensor([0.0, 1.0]), + enable_grad=True, + ) + # Only the continuous modality should have a gradient + loss = result[0].sum() + loss.backward() + self.assertIsNotNone(x_cont.grad) + self.assertIsNone(x_disc.grad) + + # ------------------------------------------------------------------ + # Divergence‑free term test (non‑zero) + # ------------------------------------------------------------------ + def test_divergence_free(self): + # Use a mock model that returns zero logits for the discrete modality + mock_model = MagicMock() + mock_model.forward = MagicMock() + mock_model.forward.return_value = [ + torch.zeros(1, 1), + torch.zeros(1, 1, self.vocab_size), + ] + + solver = MultimodalSolver( + model=mock_model, + modality_configs=[self.continuous_cfg, self.discrete_cfg], + source_distribution_p=self.source_p, + ) + x_cont = torch.tensor([[0.0]]) + x_disc = torch.tensor([[0]]) + # Use a constant divergence‑free term + result = solver.sample( + x_init=[x_cont, x_disc], + step_size=0.1, + div_free=0.5, + time_grid=torch.tensor([0.0, 1.0]), + ) + # With a non‑zero div_free, the solver should not raise an assertion. + # The exact numeric value is not critical; we just ensure the call succeeds. + self.assertIsInstance(result, list) + self.assertEqual(len(result), 2) + + # ------------------------------------------------------------------ + # Error handling tests + # ------------------------------------------------------------------ + def test_mismatched_initial_states(self): + solver = MultimodalSolver( + model=self.model, + modality_configs=[self.continuous_cfg, self.discrete_cfg], + ) + # Provide only one initial state instead of two + with self.assertRaises(ValueError): + solver.sample( + x_init=[torch.tensor([[0.0]])], + step_size=0.1, + time_grid=torch.tensor([0.0, 1.0]), + ) + + def test_invalid_modality_type(self): + # Create a bad config list + bad_cfg = [{"type": "unknown"}] + with self.assertRaises(ValueError): + MultimodalSolver( + model=self.model, + modality_configs=bad_cfg, + ) + + def test_missing_path_for_discrete(self): + bad_cfg = [{"type": "discrete"}] # No 'path' key + with self.assertRaises(ValueError): + MultimodalSolver( + model=self.model, + modality_configs=bad_cfg, + ) + + def test_non_callable_model(self): + with self.assertRaises(TypeError): + MultimodalSolver( + model=123, # Not callable + modality_configs=[self.continuous_cfg], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/utils/test_multimodal.py b/tests/utils/test_multimodal.py new file mode 100644 index 0000000..d41770b --- /dev/null +++ b/tests/utils/test_multimodal.py @@ -0,0 +1,263 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the CC-by-NC license found in the +# LICENSE file in the root directory of this source tree. +import unittest +from unittest.mock import patch + +import torch +from flow_matching.path import AffineProbPath, MixtureDiscreteProbPath +from flow_matching.path.scheduler import CondOTScheduler, PolynomialConvexScheduler + +from flow_matching.utils.multimodal import _default_continuous_loss, Flow +from torch import nn + + +class DummyModel(nn.Module): + """Model that returns logits for discrete and scaled inputs for continuous.""" + + def __init__(self, num_classes: int = 5): + super().__init__() + self.num_classes = num_classes + + def forward(self, xs, t, **kwargs): + outputs = [] + for x in xs: + if x.dtype == torch.long: + batch = x.shape[0] + # Return random logits for discrete modality + outputs.append(torch.randn(batch, self.num_classes)) + else: + # Return a simple transformation for continuous modality + outputs.append(x * 2.0) + return outputs + + +class DummyMultimodalSolver: + """Mock solver that records arguments and returns predefined samples.""" + + def __init__(self, model, modality_configs, model_sampling_fn=None): + self.model = model + self.modality_configs = modality_configs + self.model_sampling_fn = model_sampling_fn + self.called_with = {} + + def sample(self, **kwargs): + self.called_with = kwargs + # Return a list of tensors matching the number of modalities + return [torch.tensor([1]), torch.tensor([2.0])] + + +class TestFlow(unittest.TestCase): + def setUp(self): + self.num_classes = 5 + self.discrete_path = MixtureDiscreteProbPath( + scheduler=PolynomialConvexScheduler(n=2.0) + ) + self.continuous_path = AffineProbPath(scheduler=CondOTScheduler()) + self.modalities = { + "disc": {"path": self.discrete_path}, + "cont": {"path": self.continuous_path}, + } + self.model = DummyModel(num_classes=self.num_classes) + self.flow = Flow(model=self.model, modalities=self.modalities) + + def test_init_paths_and_losses(self): + # Paths should be stored correctly + self.assertIn("disc", self.flow.paths) + self.assertIn("cont", self.flow.paths) + self.assertIs(self.flow.paths["disc"], self.discrete_path) + self.assertIs(self.flow.paths["cont"], self.continuous_path) + + # Loss functions: discrete should be MixturePathGeneralizedKL (callable) + self.assertTrue(callable(self.flow.loss_fns["disc"])) + # Continuous should use the default continuous loss + self.assertIs(self.flow.loss_fns["cont"], _default_continuous_loss) + + def test_training_loss_computation(self): + batch = 3 + # Discrete tensors (int64) + x1_disc = torch.randint(0, self.num_classes, (batch,)) + x_t_disc = torch.randint(0, self.num_classes, (batch,)) + # Continuous tensors (float32) + x1_cont = torch.randn(batch, 2) + x_t_cont = torch.randn(batch, 2) + dx_t_cont = torch.randn(batch, 2) + # Assemble inputs matching modality order (disc, cont) + x_1 = [x1_disc, x1_cont] + x_t = [x_t_disc, x_t_cont] + dx_t = [None, dx_t_cont] + t = [torch.rand(batch), torch.rand(batch)] + + total_loss, loss_dict = self.flow.training_loss(x_1, x_t, dx_t, t) + + # Total loss should be a scalar tensor + self.assertIsInstance(total_loss, torch.Tensor) + self.assertEqual(total_loss.dim(), 0) + + # Loss dict should contain both modalities + self.assertSetEqual(set(loss_dict.keys()), {"disc", "cont"}) + # Each entry should be a scalar tensor + for loss in loss_dict.values(): + self.assertIsInstance(loss, torch.Tensor) + self.assertEqual(loss.mean().dim(), 0) + + # Total loss should equal sum of individual losses + summed = sum(loss.mean() for loss in loss_dict.values()) + self.assertTrue(torch.allclose(total_loss, summed)) + + def test_training_loss_mismatched_lengths(self): + batch = 2 + x1_disc = torch.randint(0, self.num_classes, (batch,)) + x_t_disc = torch.randint(0, self.num_classes, (batch,)) + # x1_cont = torch.randn(batch, 2) + # x_t_cont = torch.randn(batch, 2) + # dx_t_cont = torch.randn(batch, 2) + + # Omit the continuous modality to trigger assertion + x_1 = [x1_disc] + x_t = [x_t_disc] + dx_t = [None] + t = [torch.rand(batch)] + + with self.assertRaises(AssertionError): + self.flow.training_loss(x_1, x_t, dx_t, t) + + def test_sample_dtype_validation_and_output(self): + batch = 4 + # Correct dtypes + x_init_disc = torch.randint(0, self.num_classes, (batch,)) + x_init_cont = torch.randn(batch, 2) + + with patch( + "flow_matching.utils.multimodal.MultimodalSolver", + DummyMultimodalSolver, + ): + self.flow = Flow( + model=self.model, modalities=self.modalities + ) # Reinitialize to use dummy solver + samples = self.flow.sample([x_init_disc, x_init_cont], steps=5) + + # Should receive the dummy solver's output + self.assertEqual(len(samples), 2) + self.assertTrue(torch.equal(samples[0], torch.tensor([1]))) + self.assertTrue(torch.equal(samples[1], torch.tensor([2.0]))) + + def test_sample_wrong_dtype_raises(self): + batch = 3 + # Wrong dtype for discrete modality (float instead of long) + x_init_disc = torch.randn(batch, dtype=torch.float32) + x_init_cont = torch.randn(batch, 2) + + with self.assertRaises(AssertionError): + self.flow.sample([x_init_disc, x_init_cont], steps=5) + + def test_custom_loss_weights(self): + # Define modalities with custom loss weights + modalities = { + "disc": {"path": self.discrete_path, "weight": 0.5}, + "cont": {"path": self.continuous_path, "weight": 2.0}, + } + flow = Flow(model=self.model, modalities=modalities) + + # Prepare inputs + batch = 3 + x1_disc = torch.randint(0, self.num_classes, (batch,)) + x_t_disc = torch.randint(0, self.num_classes, (batch,)) + x1_cont = torch.randn(batch, 2) + x_t_cont = torch.randn(batch, 2) + dx_t_cont = torch.randn(batch, 2) + x_1 = [x1_disc, x1_cont] + x_t = [x_t_disc, x_t_cont] + dx_t = [None, dx_t_cont] + t = [torch.rand(batch), torch.rand(batch)] + + total_loss, loss_dict = flow.training_loss(x_1, x_t, dx_t, t) + + # Compute expected weighted total loss + expected_total = loss_dict["disc"].mean() + loss_dict["cont"].mean() + self.assertTrue(torch.allclose(total_loss, expected_total)) + + # Verify that loss_weights are stored correctly + self.assertEqual(flow.loss_weights["disc"], 0.5) + self.assertEqual(flow.loss_weights["cont"], 2.0) + + def test_training_loss_x1_prediction_true(self): + # Define a custom continuous loss that returns the target tensor. + def custom_continuous_loss(pred, target, reduction="none"): + # Return the target directly to verify it's used. + return target + + # Set up modalities with x_1_prediction enabled for the continuous path. + modalities = { + "disc": {"path": self.discrete_path}, + "cont": { + "path": self.continuous_path, + "loss": custom_continuous_loss, + "x_1_prediction": True, + }, + } + flow = Flow(model=self.model, modalities=modalities) + + # Prepare inputs. + batch = 3 + x1_disc = torch.randint(0, self.num_classes, (batch,)) + x_t_disc = torch.randint(0, self.num_classes, (batch,)) + x1_cont = torch.randn(batch, 2) + x_t_cont = torch.randn(batch, 2) + dx_t_cont = torch.randn( + batch, 2 + ) # Should be ignored due to x_1_prediction=True + x_1 = [x1_disc, x1_cont] + x_t = [x_t_disc, x_t_cont] + dx_t = [None, dx_t_cont] + t = [torch.rand(batch), torch.rand(batch)] + + total_loss, loss_dict = flow.training_loss(x_1, x_t, dx_t, t) + + # The continuous loss should have used x1_cont as the target. + self.assertTrue(torch.allclose(loss_dict["cont"], x1_cont)) + # Total loss should be sum of discrete loss mean and x1_cont mean. + expected_total = loss_dict["disc"].mean() + loss_dict["cont"].mean() + self.assertTrue(torch.allclose(total_loss, expected_total)) + + def test_training_loss_with_logits_argument(self): + batch = 3 + # Discrete tensors (int64) + x1_disc = torch.randint(0, self.num_classes, (batch,)) + x_t_disc = torch.randint(0, self.num_classes, (batch,)) + # Continuous tensors (float32) + x1_cont = torch.randn(batch, 2) + x_t_cont = torch.randn(batch, 2) + dx_t_cont = torch.randn(batch, 2) + x_1 = [x1_disc, x1_cont] + x_t = [x_t_disc, x_t_cont] + dx_t = [None, dx_t_cont] + t = [torch.rand(batch), torch.rand(batch)] + + # Deterministic logits for discrete and continuous modalities + logits_disc = torch.full((batch, self.num_classes), 0.5) + logits_cont = torch.full_like(dx_t_cont, 0.1) + logits = [logits_disc, logits_cont] + + # Ensure model forward is not called when logits are provided + with patch.object( + self.flow.model, + "forward", + side_effect=AssertionError("Model forward should not be called"), + ): + total_loss, loss_dict = self.flow.training_loss( + x_1, x_t, dx_t, t, model_output=logits + ) + + # Verify total loss is scalar and matches sum of individual losses + self.assertIsInstance(total_loss, torch.Tensor) + self.assertEqual(total_loss.dim(), 0) + self.assertSetEqual(set(loss_dict.keys()), {"disc", "cont"}) + summed = sum(loss.mean() for loss in loss_dict.values()) + self.assertTrue(torch.allclose(total_loss, summed)) + + +if __name__ == "__main__": + unittest.main()