diff --git a/onetime/uinHash-migration.py b/onetime/uinHash-migration.py new file mode 100644 index 00000000..b4cf22b2 --- /dev/null +++ b/onetime/uinHash-migration.py @@ -0,0 +1,83 @@ +import json +import boto3 +import logging +from botocore.exceptions import ClientError + +# --- Configuration --- +SOURCE_TABLE_NAME = "infra-core-api-uin-mapping" +DESTINATION_TABLE_NAME = "infra-core-api-user-info" +DESTINATION_ID_SUFFIX = "@illinois.edu" + +# --- Logging Setup --- +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) + + +def migrate_uin_hashes(): + """ + Scans the source table for netId and uinHash mappings and updates + the corresponding user records in the destination table. + """ + try: + dynamodb = boto3.resource("dynamodb") + destination_table = dynamodb.Table(DESTINATION_TABLE_NAME) + + # A paginator is used to handle scanning tables of any size + paginator = dynamodb.meta.client.get_paginator("scan") + page_iterator = paginator.paginate(TableName=SOURCE_TABLE_NAME) + + scanned_count = 0 + updated_count = 0 + logging.info( + f"Starting migration from '{SOURCE_TABLE_NAME}' to '{DESTINATION_TABLE_NAME}'" + ) + + for page in page_iterator: + for item in page.get("Items", []): + scanned_count += 1 + net_id = item.get("netId") + uin_hash = item.get("uinHash") + + # Validate that the necessary fields exist + if not net_id or not uin_hash: + logging.warning( + f"Skipping item with missing 'netId' or 'uinHash': {item}" + ) + continue + + # Construct the primary key and update parameters for the destination table + destination_pk_id = f"{net_id}{DESTINATION_ID_SUFFIX}" + update_expression = "SET uinHash = :uh, netId = :ne" + expression_attribute_values = {":uh": uin_hash, ":ne": net_id} + + # Update the item in the destination DynamoDB table + try: + destination_table.update_item( + Key={"id": destination_pk_id}, + UpdateExpression=update_expression, + ExpressionAttributeValues=expression_attribute_values, + ) + updated_count += 1 + if updated_count % 100 == 0: + logging.info( + f"Scanned {scanned_count} items, updated {updated_count} so far..." + ) + except ClientError as e: + logging.error( + f"Failed to update item with id '{destination_pk_id}': {e}" + ) + + logging.info("--- Script Finished ---") + logging.info(f"Total items scanned from source: {scanned_count}") + logging.info(f"Total items updated in destination: {updated_count}") + + except ClientError as e: + # This will catch errors like table not found, or credential issues + logging.critical(f"A critical AWS error occurred: {e}") + except Exception as e: + logging.critical(f"An unexpected error occurred: {e}") + + +if __name__ == "__main__": + migrate_uin_hashes() diff --git a/src/api/functions/sync.ts b/src/api/functions/sync.ts new file mode 100644 index 00000000..190b782f --- /dev/null +++ b/src/api/functions/sync.ts @@ -0,0 +1,46 @@ +import { + UpdateItemCommand, + type DynamoDBClient, +} from "@aws-sdk/client-dynamodb"; +import { genericConfig } from "common/config.js"; + +export interface SyncFullProfileInputs { + uinHash: string; + netId: string; + firstName: string; + lastName: string; + dynamoClient: DynamoDBClient; +} + +export async function syncFullProfile({ + uinHash, + netId, + firstName, + lastName, + dynamoClient, +}: SyncFullProfileInputs) { + return dynamoClient.send( + new UpdateItemCommand({ + TableName: genericConfig.UserInfoTable, + Key: { + id: { S: `${netId}@illinois.edu` }, + }, + UpdateExpression: + "SET #uinHash = :uinHash, #netId = :netId, #updatedAt = :updatedAt, #firstName = :firstName, #lastName = :lastName", + ExpressionAttributeNames: { + "#uinHash": "uinHash", + "#netId": "netId", + "#updatedAt": "updatedAt", + "#firstName": "firstName", + "#lastName": "lastName", + }, + ExpressionAttributeValues: { + ":uinHash": { S: uinHash }, + ":netId": { S: netId }, + ":firstName": { S: firstName }, + ":lastName": { S: lastName }, + ":updatedAt": { S: new Date().toISOString() }, + }, + }), + ); +} diff --git a/src/api/functions/uin.ts b/src/api/functions/uin.ts index ca3b2165..45e79aaf 100644 --- a/src/api/functions/uin.ts +++ b/src/api/functions/uin.ts @@ -1,4 +1,8 @@ -import { DynamoDBClient, PutItemCommand } from "@aws-sdk/client-dynamodb"; +import { + DynamoDBClient, + PutItemCommand, + UpdateItemCommand, +} from "@aws-sdk/client-dynamodb"; import { marshall } from "@aws-sdk/util-dynamodb"; import { argon2id, hash } from "argon2"; import { genericConfig } from "common/config.js"; @@ -160,13 +164,23 @@ export async function saveHashedUserUin({ }: SaveHashedUserUin) { const uinHash = await getHashedUserUin({ uiucAccessToken, pepper }); await dynamoClient.send( - new PutItemCommand({ - TableName: genericConfig.UinHashTable, - Item: marshall({ - uinHash, - netId, - updatedAt: new Date().toISOString(), - }), + new UpdateItemCommand({ + TableName: genericConfig.UserInfoTable, + Key: { + id: { S: `${netId}@illinois.edu` }, + }, + UpdateExpression: + "SET #uinHash = :uinHash, #netId = :netId, #updatedAt = :updatedAt", + ExpressionAttributeNames: { + "#uinHash": "uinHash", + "#netId": "netId", + "#updatedAt": "updatedAt", + }, + ExpressionAttributeValues: { + ":uinHash": { S: uinHash }, + ":netId": { S: netId }, + ":updatedAt": { S: new Date().toISOString() }, + }, }), ); } diff --git a/src/api/routes/syncIdentity.ts b/src/api/routes/syncIdentity.ts index eba75ba4..07ce9b48 100644 --- a/src/api/routes/syncIdentity.ts +++ b/src/api/routes/syncIdentity.ts @@ -8,7 +8,7 @@ import rateLimiter from "api/plugins/rateLimiter.js"; import { FastifyZodOpenApiTypeProvider } from "fastify-zod-openapi"; import * as z from "zod/v4"; import { notAuthenticatedError, withTags } from "api/components/index.js"; -import { verifyUiucAccessToken, saveHashedUserUin } from "api/functions/uin.js"; +import { verifyUiucAccessToken, getHashedUserUin } from "api/functions/uin.js"; import { getRoleCredentials } from "api/functions/sts.js"; import { SecretsManagerClient } from "@aws-sdk/client-secrets-manager"; import { genericConfig, roleArns } from "common/config.js"; @@ -18,6 +18,7 @@ import { patchUserProfile, resolveEmailToOid, } from "api/functions/entraId.js"; +import { syncFullProfile } from "api/functions/sync.js"; const syncIdentityPlugin: FastifyPluginAsync = async (fastify, _options) => { const getAuthorizedClients = async () => { @@ -98,11 +99,16 @@ const syncIdentityPlugin: FastifyPluginAsync = async (fastify, _options) => { message: "ID token could not be parsed.", }); } - await saveHashedUserUin({ + const uinHash = await getHashedUserUin({ uiucAccessToken: accessToken, pepper: fastify.secretConfig.UIN_HASHING_SECRET_PEPPER, - dynamoClient: fastify.dynamoClient, + }); + await syncFullProfile({ + uinHash, + firstName: givenName, + lastName: surname, netId, + dynamoClient: fastify.dynamoClient, }); let isPaidMember = await checkPaidMembershipFromRedis( netId, diff --git a/src/api/routes/v2/membership.ts b/src/api/routes/v2/membership.ts index b0bce7aa..d37e3c13 100644 --- a/src/api/routes/v2/membership.ts +++ b/src/api/routes/v2/membership.ts @@ -22,7 +22,7 @@ import { withRoles, withTags, } from "api/components/index.js"; -import { verifyUiucAccessToken, saveHashedUserUin } from "api/functions/uin.js"; +import { verifyUiucAccessToken, getHashedUserUin } from "api/functions/uin.js"; import { getKey, setKey } from "api/functions/redisCache.js"; import { getEntraIdToken } from "api/functions/entraId.js"; import { genericConfig, roleArns } from "common/config.js"; @@ -31,6 +31,7 @@ import { SecretsManagerClient } from "@aws-sdk/client-secrets-manager"; import { BatchGetItemCommand, DynamoDBClient } from "@aws-sdk/client-dynamodb"; import { AppRoles } from "common/roles.js"; import { marshall, unmarshall } from "@aws-sdk/util-dynamodb"; +import { syncFullProfile } from "api/functions/sync.js"; const membershipV2Plugin: FastifyPluginAsync = async (fastify, _options) => { const getAuthorizedClients = async () => { @@ -115,11 +116,16 @@ const membershipV2Plugin: FastifyPluginAsync = async (fastify, _options) => { }); } request.log.debug("Saving user hashed UIN!"); - const saveHashPromise = saveHashedUserUin({ + const uinHash = await getHashedUserUin({ uiucAccessToken: accessToken, pepper: fastify.secretConfig.UIN_HASHING_SECRET_PEPPER, - dynamoClient: fastify.dynamoClient, + }); + const savePromise = syncFullProfile({ + uinHash, + firstName: givenName, + lastName: surname, netId, + dynamoClient: fastify.dynamoClient, }); let isPaidMember = await checkPaidMembershipFromRedis( netId, @@ -132,7 +138,7 @@ const membershipV2Plugin: FastifyPluginAsync = async (fastify, _options) => { fastify.dynamoClient, ); } - await saveHashPromise; + await savePromise; request.log.debug("Saved user hashed UIN!"); if (isPaidMember) { throw new ValidationError({ diff --git a/src/common/config.ts b/src/common/config.ts index e1d55f99..d15fc3d1 100644 --- a/src/common/config.ts +++ b/src/common/config.ts @@ -55,7 +55,7 @@ export type GenericConfigType = { TestingCredentialsSecret: string; UinHashingSecret: string; UinExtendedAttributeName: string; - UinHashTable: string; + UserInfoTable: string; }; type EnvironmentConfigType = { @@ -96,7 +96,7 @@ const genericConfig: GenericConfigType = { TestingCredentialsSecret: "infra-core-api-testing-credentials", UinHashingSecret: "infra-core-api-uin-pepper", UinExtendedAttributeName: "extension_a70c2e1556954056a6a8edfb1f42f556_uiucEduUIN", - UinHashTable: "infra-core-api-uin-mapping", + UserInfoTable: "infra-core-api-user-info" } as const; const environmentConfig: EnvironmentConfigType = { diff --git a/terraform/modules/dynamo/main.tf b/terraform/modules/dynamo/main.tf index a784e964..fcd0470f 100644 --- a/terraform/modules/dynamo/main.tf +++ b/terraform/modules/dynamo/main.tf @@ -1,3 +1,15 @@ +resource "null_resource" "onetime_uin_migration" { + depends_on = [aws_dynamodb_table.user_info] + provisioner "local-exec" { + command = <<-EOT + set -e + python uinHash-migration.py + EOT + interpreter = ["bash", "-c"] + working_dir = "${path.module}/../../../onetime/" + } +} + resource "aws_dynamodb_table" "app_audit_log" { billing_mode = "PAY_PER_REQUEST" name = "${var.ProjectId}-audit-log" @@ -190,6 +202,28 @@ resource "aws_dynamodb_table" "iam_user_roles" { } } +resource "aws_dynamodb_table" "user_info" { + billing_mode = "PAY_PER_REQUEST" + name = "${var.ProjectId}-user-info" + deletion_protection_enabled = true + hash_key = "id" + point_in_time_recovery { + enabled = true + } + attribute { + name = "id" + type = "S" + } + attribute { + name = "uinHash" + type = "S" + } + global_secondary_index { + name = "UinHashIndex" + hash_key = "uinHash" + projection_type = "KEYS_ONLY" + } +} resource "aws_dynamodb_table" "events" { billing_mode = "PAY_PER_REQUEST" @@ -298,7 +332,7 @@ resource "aws_dynamodb_table" "cache" { resource "aws_dynamodb_table" "app_uin_records" { billing_mode = "PAY_PER_REQUEST" name = "${var.ProjectId}-uin-mapping" - deletion_protection_enabled = true + deletion_protection_enabled = false hash_key = "uinHash" point_in_time_recovery { enabled = true diff --git a/terraform/modules/lambdas/main.tf b/terraform/modules/lambdas/main.tf index 9e8dd1dd..23bb7306 100644 --- a/terraform/modules/lambdas/main.tf +++ b/terraform/modules/lambdas/main.tf @@ -268,16 +268,17 @@ resource "aws_iam_policy" "shared_iam_policy" { ] }, { - Sid = "DynamoDBUINAccess", + Sid = "DynamoDBUserInfoAccess", Effect = "Allow", Action = [ "dynamodb:PutItem", + "dynamodb:UpdateItem", "dynamodb:DescribeTable", "dynamodb:Query", ], Resource = [ - "arn:aws:dynamodb:${data.aws_region.current.region}:${data.aws_caller_identity.current.account_id}:table/infra-core-api-uin-mapping", - "arn:aws:dynamodb:${data.aws_region.current.region}:${data.aws_caller_identity.current.account_id}:table/infra-core-api-uin-mapping/index/*", + "arn:aws:dynamodb:${data.aws_region.current.region}:${data.aws_caller_identity.current.account_id}:table/infra-core-api-user-info", + "arn:aws:dynamodb:${data.aws_region.current.region}:${data.aws_caller_identity.current.account_id}:table/infra-core-api-user-info/index/*", ] }, {