feat(bigquery): Add support for custom timezones and timestamps (#3859) · googleapis/java-bigquery@e5467c9 (original) (raw)
`@@ -19,9 +19,12 @@
`
19
19
`import static com.google.common.base.Preconditions.checkArgument;
`
20
20
`import static com.google.common.base.Strings.isNullOrEmpty;
`
21
21
``
``
22
`+
import com.google.api.core.ApiFunction;
`
22
23
`import com.google.api.services.bigquery.model.ExternalDataConfiguration;
`
23
24
`import com.google.api.services.bigquery.model.Table;
`
24
25
`import com.google.auto.value.AutoValue;
`
``
26
`+
import com.google.cloud.StringEnumType;
`
``
27
`+
import com.google.cloud.StringEnumValue;
`
25
28
`import com.google.common.base.Function;
`
26
29
`import com.google.common.collect.ImmutableList;
`
27
30
`import java.util.List;
`
`@@ -57,6 +60,46 @@ public ExternalDataConfiguration apply(ExternalTableDefinition tableInfo) {
`
57
60
``
58
61
`private static final long serialVersionUID = -5951580238459622025L;
`
59
62
``
``
63
`+
public static final class SourceColumnMatch extends StringEnumValue {
`
``
64
`+
private static final long serialVersionUID = 818920627219751207L;
`
``
65
`+
private static final ApiFunction<String, SourceColumnMatch> CONSTRUCTOR =
`
``
66
`+
new ApiFunction<String, SourceColumnMatch>() {
`
``
67
`+
@Override
`
``
68
`+
public SourceColumnMatch apply(String constant) {
`
``
69
`+
return new SourceColumnMatch(constant);
`
``
70
`+
}
`
``
71
`+
};
`
``
72
+
``
73
`+
private static final StringEnumType type =
`
``
74
`+
new StringEnumType(SourceColumnMatch.class, CONSTRUCTOR);
`
``
75
+
``
76
`+
public static final SourceColumnMatch POSITION = type.createAndRegister("POSITION");
`
``
77
+
``
78
`+
public static final SourceColumnMatch NAME = type.createAndRegister("NAME");
`
``
79
+
``
80
`+
private SourceColumnMatch(String constant) {
`
``
81
`+
super(constant);
`
``
82
`+
}
`
``
83
+
``
84
`+
/**
`
``
85
`+
- Get the SourceColumnMatch for the given String constant, and throw an exception if the
`
``
86
`+
- constant is not recognized.
`
``
87
`+
*/
`
``
88
`+
public static SourceColumnMatch valueOfStrict(String constant) {
`
``
89
`+
return type.valueOfStrict(constant);
`
``
90
`+
}
`
``
91
+
``
92
`+
/** Get the SourceColumnMatch for the given String constant, and allow unrecognized values. */
`
``
93
`+
public static SourceColumnMatch valueOf(String constant) {
`
``
94
`+
return type.valueOf(constant);
`
``
95
`+
}
`
``
96
+
``
97
`+
/** Return the known values for SourceColumnMatch. */
`
``
98
`+
public static SourceColumnMatch[] values() {
`
``
99
`+
return type.values();
`
``
100
`+
}
`
``
101
`+
}
`
``
102
+
60
103
`@AutoValue.Builder
`
61
104
`public abstract static class Builder
`
62
105
`extends TableDefinition.Builder<ExternalTableDefinition, Builder> {
`
`@@ -215,6 +258,45 @@ public Builder setMaxStaleness(String maxStaleness) {
`
215
258
``
216
259
`abstract Builder setMaxStalenessInner(String maxStaleness);
`
217
260
``
``
261
`+
/**
`
``
262
`+
- Time zone used when parsing timestamp values that do not have specific time zone information
`
``
263
`+
- (e.g. 2024-04-20 12:34:56). The expected format is a IANA timezone string (e.g.
`
``
264
`+
- America/Los_Angeles).
`
``
265
`+
*/
`
``
266
`+
public abstract Builder setTimeZone(String timeZone);
`
``
267
+
``
268
`+
/** Format used to parse DATE values. Supports C-style and SQL-style values. */
`
``
269
`+
public abstract Builder setDateFormat(String dateFormat);
`
``
270
+
``
271
`+
/** Format used to parse DATETIME values. Supports C-style and SQL-style values. */
`
``
272
`+
public abstract Builder setDatetimeFormat(String datetimeFormat);
`
``
273
+
``
274
`+
/** Format used to parse TIME values. Supports C-style and SQL-style values. */
`
``
275
`+
public abstract Builder setTimeFormat(String timeFormat);
`
``
276
+
``
277
`+
/** Format used to parse TIMESTAMP values. Supports C-style and SQL-style values. */
`
``
278
`+
public abstract Builder setTimestampFormat(String timestampFormat);
`
``
279
+
``
280
`+
/**
`
``
281
`+
- Controls the strategy used to match loaded columns to the schema. If not set, a sensible
`
``
282
`+
- default is chosen based on how the schema is provided. If autodetect is used, then columns
`
``
283
`+
- are matched by name. Otherwise, columns are matched by position. This is done to keep the
`
``
284
`+
- behavior backward-compatible. Acceptable values are: POSITION - matches by position. This
`
``
285
`+
- assumes that the columns are ordered the same way as the schema. NAME - matches by name. This
`
``
286
`+
- reads the header row as column names and reorders columns to match the field names in the
`
``
287
`+
- schema.
`
``
288
`+
*/
`
``
289
`+
public abstract Builder setSourceColumnMatch(SourceColumnMatch sourceColumnMatch);
`
``
290
+
``
291
`+
/**
`
``
292
`+
- A list of strings represented as SQL NULL value in a CSV file. null_marker and null_markers
`
``
293
`+
- can't be set at the same time. If null_marker is set, null_markers has to be not set. If
`
``
294
`+
- null_markers is set, null_marker has to be not set. If both null_marker and null_markers are
`
``
295
`+
- set at the same time, a user error would be thrown. Any strings listed in null_markers,
`
``
296
`+
- including empty string would be interpreted as SQL NULL. This applies to all column types.
`
``
297
`+
*/
`
``
298
`+
public abstract Builder setNullMarkers(List nullMarkers);
`
``
299
+
218
300
`/** Creates an {@code ExternalTableDefinition} object. */
`
219
301
`@Override
`
220
302
`public abstract ExternalTableDefinition build();
`
`@@ -373,6 +455,37 @@ public HivePartitioningOptions getHivePartitioningOptions() {
`
373
455
`return getHivePartitioningOptionsInner();
`
374
456
` }
`
375
457
``
``
458
`+
/**
`
``
459
`+
- Returns the time zone used when parsing timestamp values that don't have specific time zone
`
``
460
`+
- information.
`
``
461
`+
*/
`
``
462
`+
@Nullable
`
``
463
`+
public abstract String getTimeZone();
`
``
464
+
``
465
`+
/** Returns the format used to parse DATE values. */
`
``
466
`+
@Nullable
`
``
467
`+
public abstract String getDateFormat();
`
``
468
+
``
469
`+
/** Returns the format used to parse DATETIME values. */
`
``
470
`+
@Nullable
`
``
471
`+
public abstract String getDatetimeFormat();
`
``
472
+
``
473
`+
/** Returns the format used to parse TIME values. */
`
``
474
`+
@Nullable
`
``
475
`+
public abstract String getTimeFormat();
`
``
476
+
``
477
`+
/** Returns the format used to parse TIMESTAMP values. */
`
``
478
`+
@Nullable
`
``
479
`+
public abstract String getTimestampFormat();
`
``
480
+
``
481
`+
/** Returns the strategy used to match loaded columns to the schema, either POSITION or NAME. */
`
``
482
`+
@Nullable
`
``
483
`+
public abstract SourceColumnMatch getSourceColumnMatch();
`
``
484
+
``
485
`+
/** Returns a list of strings represented as SQL NULL value in a CSV file. */
`
``
486
`+
@Nullable
`
``
487
`+
public abstract List getNullMarkers();
`
``
488
+
376
489
`@Nullable
`
377
490
`abstract HivePartitioningOptions getHivePartitioningOptionsInner();
`
378
491
``
`@@ -454,6 +567,29 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC
`
454
567
`if (getMetadataCacheMode() != null) {
`
455
568
`externalConfigurationPb.setMetadataCacheMode(getMetadataCacheMode());
`
456
569
` }
`
``
570
`+
if (getTimeZone() != null) {
`
``
571
`+
externalConfigurationPb.setTimeZone(getTimeZone());
`
``
572
`+
}
`
``
573
`+
if (getDateFormat() != null) {
`
``
574
`+
externalConfigurationPb.setDateFormat(getDateFormat());
`
``
575
`+
}
`
``
576
`+
if (getDatetimeFormat() != null) {
`
``
577
`+
externalConfigurationPb.setDatetimeFormat(getDatetimeFormat());
`
``
578
`+
}
`
``
579
`+
if (getTimeFormat() != null) {
`
``
580
`+
externalConfigurationPb.setTimeFormat(getTimeFormat());
`
``
581
`+
}
`
``
582
`+
if (getTimestampFormat() != null) {
`
``
583
`+
externalConfigurationPb.setTimestampFormat(getTimestampFormat());
`
``
584
`+
}
`
``
585
`+
if (getSourceColumnMatch() != null) {
`
``
586
`+
externalConfigurationPb
`
``
587
`+
.getCsvOptions()
`
``
588
`+
.setSourceColumnMatch(getSourceColumnMatch().toString());
`
``
589
`+
}
`
``
590
`+
if (getNullMarkers() != null) {
`
``
591
`+
externalConfigurationPb.getCsvOptions().setNullMarkers(getNullMarkers());
`
``
592
`+
}
`
457
593
``
458
594
`return externalConfigurationPb;
`
459
595
` }
`
`@@ -654,6 +790,31 @@ static ExternalTableDefinition fromPb(Table tablePb) {
`
654
790
`if (tablePb.getMaxStaleness() != null) {
`
655
791
`builder.setMaxStaleness(tablePb.getMaxStaleness());
`
656
792
` }
`
``
793
`+
if (externalDataConfiguration.getTimeZone() != null) {
`
``
794
`+
builder.setTimeZone(externalDataConfiguration.getTimeZone());
`
``
795
`+
}
`
``
796
`+
if (externalDataConfiguration.getDateFormat() != null) {
`
``
797
`+
builder.setDateFormat(externalDataConfiguration.getDateFormat());
`
``
798
`+
}
`
``
799
`+
if (externalDataConfiguration.getDatetimeFormat() != null) {
`
``
800
`+
builder.setDatetimeFormat(externalDataConfiguration.getDatetimeFormat());
`
``
801
`+
}
`
``
802
`+
if (externalDataConfiguration.getTimeFormat() != null) {
`
``
803
`+
builder.setTimeFormat(externalDataConfiguration.getTimeFormat());
`
``
804
`+
}
`
``
805
`+
if (externalDataConfiguration.getTimestampFormat() != null) {
`
``
806
`+
builder.setTimestampFormat(externalDataConfiguration.getTimestampFormat());
`
``
807
`+
}
`
``
808
`+
if (externalDataConfiguration.getCsvOptions() != null) {
`
``
809
`+
if (externalDataConfiguration.getCsvOptions().getSourceColumnMatch() != null) {
`
``
810
`+
builder.setSourceColumnMatch(
`
``
811
`+
SourceColumnMatch.valueOf(
`
``
812
`+
externalDataConfiguration.getCsvOptions().getSourceColumnMatch()));
`
``
813
`+
}
`
``
814
`+
if (externalDataConfiguration.getCsvOptions().getNullMarkers() != null) {
`
``
815
`+
builder.setNullMarkers(externalDataConfiguration.getCsvOptions().getNullMarkers());
`
``
816
`+
}
`
``
817
`+
}
`
657
818
` }
`
658
819
`return builder.build();
`
659
820
` }
`
`@@ -724,6 +885,31 @@ static ExternalTableDefinition fromExternalDataConfiguration(
`
724
885
`if (externalDataConfiguration.getMetadataCacheMode() != null) {
`
725
886
`builder.setMetadataCacheMode(externalDataConfiguration.getMetadataCacheMode());
`
726
887
` }
`
``
888
`+
if (externalDataConfiguration.getTimeZone() != null) {
`
``
889
`+
builder.setTimeZone(externalDataConfiguration.getTimeZone());
`
``
890
`+
}
`
``
891
`+
if (externalDataConfiguration.getDateFormat() != null) {
`
``
892
`+
builder.setDateFormat(externalDataConfiguration.getDateFormat());
`
``
893
`+
}
`
``
894
`+
if (externalDataConfiguration.getDatetimeFormat() != null) {
`
``
895
`+
builder.setDatetimeFormat(externalDataConfiguration.getDatetimeFormat());
`
``
896
`+
}
`
``
897
`+
if (externalDataConfiguration.getTimeFormat() != null) {
`
``
898
`+
builder.setTimeFormat(externalDataConfiguration.getTimeFormat());
`
``
899
`+
}
`
``
900
`+
if (externalDataConfiguration.getTimestampFormat() != null) {
`
``
901
`+
builder.setTimestampFormat(externalDataConfiguration.getTimeFormat());
`
``
902
`+
}
`
``
903
`+
if (externalDataConfiguration.getCsvOptions() != null) {
`
``
904
`+
if (externalDataConfiguration.getCsvOptions().getSourceColumnMatch() != null) {
`
``
905
`+
builder.setSourceColumnMatch(
`
``
906
`+
SourceColumnMatch.valueOf(
`
``
907
`+
externalDataConfiguration.getCsvOptions().getSourceColumnMatch()));
`
``
908
`+
}
`
``
909
`+
if (externalDataConfiguration.getCsvOptions().getNullMarkers() != null) {
`
``
910
`+
builder.setNullMarkers(externalDataConfiguration.getCsvOptions().getNullMarkers());
`
``
911
`+
}
`
``
912
`+
}
`
727
913
``
728
914
`return builder.build();
`
729
915
` }
`