Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion datacatalog/cloud-client/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ repositories {
}

dependencies {
compile group: 'com.google.cloud', name: 'google-cloud-datacatalog', version:'0.4.0-alpha'
compile group: 'com.google.cloud', name: 'google-cloud-datacatalog', version:'0.28.0-alpha'

testCompile group: 'com.google.truth', name: 'truth', version:'0.42'
testCompile group: 'junit', name: 'junit', version:'4.13-beta-2'
Expand Down
2 changes: 1 addition & 1 deletion datacatalog/cloud-client/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-datacatalog</artifactId>
<version>0.14.0-alpha</version>
<version>0.28.0-alpha</version>
</dependency>

<!-- Test dependencies -->
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
/*
* Copyright 2019 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.example.datacatalog;

// [START datacatalog_create_fileset_tag]

import com.google.api.gax.rpc.AlreadyExistsException;
import com.google.cloud.datacatalog.ColumnSchema;
import com.google.cloud.datacatalog.CreateEntryGroupRequest;
import com.google.cloud.datacatalog.CreateEntryRequest;
import com.google.cloud.datacatalog.Entry;
import com.google.cloud.datacatalog.EntryGroup;
import com.google.cloud.datacatalog.EntryType;
import com.google.cloud.datacatalog.GcsFilesetSpec;
import com.google.cloud.datacatalog.LocationName;
import com.google.cloud.datacatalog.Schema;
import com.google.cloud.datacatalog.v1beta1.DataCatalogClient;

import java.io.IOException;

public class CreateFilesetEntry {

/**
* Create Fileset Entry.
*
* @param projectId e.g. 'my-project'
* @param entryGroupId e.g. 'fileset_entry_group'
* @param entryId e.g. 'fileset_entry_id'
*/
public static void createEntry(String projectId, String entryGroupId, String entryId) {
// Currently, Data Catalog stores metadata in the
// us-central1 region.
String location = "us-central1";

// Initialize client that will be used to send requests. This client only needs to be created
// once, and can be reused for multiple requests. After completing all of your requests, call
// the "close" method on the client to safely clean up any remaining background resources.
try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) {
// 1. Create an Entry Group.
// -------------------------------
// Construct the EntryGroup for the EntryGroup request.
EntryGroup entryGroup = EntryGroup.newBuilder().build();

// Construct the EntryGroup request to be sent by the client.
CreateEntryGroupRequest entryGroupRequest =
CreateEntryGroupRequest.newBuilder()
.setParent(LocationName.of(projectId, location).toString())
.setEntryGroupId(entryGroupId)
.setEntryGroup(entryGroup)
.build();

// Use the client to send the API request.
EntryGroup entryGroupResponse = dataCatalogClient.createEntryGroup(entryGroupRequest);
System.out.printf("\nEntry Group created with name: %s\n", entryGroupResponse.getName());

// 2. Create a Fileset Entry.
// -------------------------------
// Construct the Entry for the Entry request.
Entry entry =
Entry.newBuilder()
.setDisplayName("My Fileset")
.setDescription("This fileset consists of ....")
.setGcsFilesetSpec(
GcsFilesetSpec.newBuilder().addFilePatterns("gs://my_bucket/*")
.build())
.setSchema(
Schema.newBuilder()
.addColumns(
ColumnSchema.newBuilder()
.setColumn("first_column")
.setType("STRING")
.setDescription(
"This columns consists of ....")
.build())
.addColumns(
ColumnSchema.newBuilder()
.setColumn("second_column")
.setType("STRING")
.setDescription(
"This columns consists of ....")
.build())
.build())
.setType(EntryType.FILESET)
.build();

// Construct the Entry request to be sent by the client.
CreateEntryRequest entryRequest =
CreateEntryRequest.newBuilder()
.setParent(entryGroupResponse.getName())
.setEntryId(entryId)
.setEntry(entry)
.build();

// Use the client to send the API request.
Entry entryResponse = dataCatalogClient.createEntry(entryRequest);
System.out.printf("\nEntry created with name: %s\n", entryResponse.getName());
} catch (AlreadyExistsException | IOException e) {
// AlreadyExistsException's are thrown if EntryGroup or Entry already exists.
// IOException's are thrown when unable to create the DataCatalogClient,
// for example an invalid Service Account path.
System.out.println("Error in create entry process:\n" + e.toString());
}
}
}
// [END datacatalog_create_fileset_tag]
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
/*
* Copyright 2019 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.example.datacatalog;

import static org.junit.Assert.assertThat;

import com.google.cloud.datacatalog.EntryGroupName;
import com.google.cloud.datacatalog.EntryName;
import com.google.cloud.datacatalog.v1beta1.DataCatalogClient;
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
import java.util.UUID;
import org.hamcrest.CoreMatchers;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;

/** Integration (system) tests for {@link CreateFilesetEntry}. */
@RunWith(JUnit4.class)
@SuppressWarnings("checkstyle:abbreviationaswordinname")
public class CreateFilesetEntryTests {

private ByteArrayOutputStream bout;

private static String ENTRY_GROUP_ID = "fileset_entry_group_"
+ UUID.randomUUID().toString().substring(0, 8);
private static String ENTRY_ID = "fileset_entry_id_"
+ UUID.randomUUID().toString().substring(0, 8);
private static String LOCATION = "us-central1";
private static String PROJECT_ID = System.getenv().get("GOOGLE_CLOUD_PROJECT");

@Before
public void setUp() {
bout = new ByteArrayOutputStream();
System.setOut(new PrintStream(bout));
}

@After
public void tearDown() {
System.setOut(null);
bout.reset();

try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) {
dataCatalogClient.deleteEntry(
EntryName.of(PROJECT_ID, LOCATION, ENTRY_GROUP_ID, ENTRY_ID).toString());
dataCatalogClient.deleteEntryGroup(
EntryGroupName.of(PROJECT_ID, LOCATION, ENTRY_GROUP_ID).toString());
} catch (Exception e) {
System.out.println("Error in cleaning up test data:\n" + e.toString());
}
}

@Test
public void testCreateFilesetEntry() {
CreateFilesetEntry.createEntry(PROJECT_ID, ENTRY_GROUP_ID, ENTRY_ID);

String output = bout.toString();

String entryGroupTemplate =
"Entry Group created with name: projects/%s/locations/us-central1/entryGroups/%s";
assertThat(
output,
CoreMatchers.containsString(String.format(entryGroupTemplate, PROJECT_ID,
ENTRY_GROUP_ID)));

String entryTemplate =
"Entry created with name: projects/%s/locations/us-central1/entryGroups/%s/entries/%s";
assertThat(
output,
CoreMatchers.containsString(
String.format(entryTemplate, PROJECT_ID, ENTRY_GROUP_ID, ENTRY_ID)));
}
}