Retry for API request failure

[y-ken]

issue

Please add option to retry 502 Bad Gateway or Connect Timeout for TD-API

env

Embulk v0.9.17
embulk-input-td (0.2.1)

error log

in this case, I have used job_id for loading result

2020-07-15 23:43:00.430 +0000: Embulk v0.9.17
2020-07-15 23:43:01.010 +0000 [WARN] (main): DEPRECATION: JRuby org.jruby.embed.ScriptingContainer is directly injected.
2020-07-15 23:43:03.233 +0000 [INFO] (main): Gem's home and path are set by default: "/home/****/.embulk/lib/gems"
2020-07-15 23:43:05.098 +0000 [INFO] (main): Started Embulk v0.9.17
2020-07-15 23:43:05.157 +0000 [INFO] (0001:transaction): Loaded plugin embulk-input-td (0.2.1)
2020-07-15 23:43:05.187 +0000 [INFO] (0001:transaction): Loaded plugin embulk-output-postgresql (0.8.2)
2020-07-15 23:43:05.218 +0000 [INFO] (0001:transaction): td-client version: 0.7.41
2020-07-15 23:43:05.222 +0000 [WARN] (0001:transaction): config file /home/***/.td/td.conf is not found
2020-07-15 23:43:05.238 +0000 [INFO] (0001:transaction): Logging initialized @5159ms
2020-07-15 23:43:05.359 +0000 [INFO] (0001:transaction): Confirm that job 117542*** finished
2020-07-15 23:43:20.396 +0000 [WARN] (0001:transaction): API request failed
java.util.concurrent.ExecutionException: java.net.SocketTimeoutException: Connect Timeout
 at org.eclipse.jetty.client.util.FutureResponseListener.getResult(FutureResponseListener.java:118) ~[jetty-client-9.2.22.v20170606.jar:9.2.22.v20170606]
 at org.eclipse.jetty.client.util.FutureResponseListener.get(FutureResponseListener.java:101) ~[jetty-client-9.2.22.v20170606.jar:9.2.22.v20170606]
 at com.treasuredata.client.TDHttpClient$DefaultContentHandler.submit(TDHttpClient.java:746) ~[td-client-0.7.41.jar:0.7.41]
 at com.treasuredata.client.TDHttpClient$DefaultContentHandler.submit(TDHttpClient.java:721) ~[td-client-0.7.41.jar:0.7.41]
 at com.treasuredata.client.TDHttpClient.submitRequest(TDHttpClient.java:374) [td-client-0.7.41.jar:0.7.41]
 at com.treasuredata.client.TDHttpClient.call(TDHttpClient.java:628) [td-client-0.7.41.jar:0.7.41]
 at com.treasuredata.client.TDHttpClient.call(TDHttpClient.java:594) [td-client-0.7.41.jar:0.7.41]
 at com.treasuredata.client.TDClient.doGet(TDClient.java:193) [td-client-0.7.41.jar:0.7.41]
 at com.treasuredata.client.TDClient.jobStatus(TDClient.java:618) [td-client-0.7.41.jar:0.7.41]
 at org.embulk.input.td.TdInputPlugin.waitJobCompletion(TdInputPlugin.java:177) [embulk-input-td-0.2.1.jar:na]
 at org.embulk.input.td.TdInputPlugin.getTdJob(TdInputPlugin.java:155) [embulk-input-td-0.2.1.jar:na]
 at org.embulk.input.td.TdInputPlugin.transaction(TdInputPlugin.java:77) [embulk-input-td-0.2.1.jar:na]
 at org.embulk.exec.BulkLoader.doRun(BulkLoader.java:507) [embulk-0.9.17:0.9.17]
 at org.embulk.exec.BulkLoader.access$000(BulkLoader.java:35) [embulk-0.9.17:0.9.17]
 at org.embulk.exec.BulkLoader$1.run(BulkLoader.java:353) [embulk-0.9.17:0.9.17]
 at org.embulk.exec.BulkLoader$1.run(BulkLoader.java:350) [embulk-0.9.17:0.9.17]
 at org.embulk.spi.Exec.doWith(Exec.java:22) [embulk-0.9.17:0.9.17]
 at org.embulk.exec.BulkLoader.run(BulkLoader.java:350) [embulk-0.9.17:0.9.17]
 at org.embulk.EmbulkEmbed.run(EmbulkEmbed.java:178) [embulk-0.9.17:0.9.17]
 at org.embulk.EmbulkRunner.runInternal(EmbulkRunner.java:292) [embulk-0.9.17:0.9.17]
 at org.embulk.EmbulkRunner.run(EmbulkRunner.java:156) [embulk-0.9.17:0.9.17]
 at org.embulk.cli.EmbulkRun.runSubcommand(EmbulkRun.java:433) [embulk-0.9.17:0.9.17]
 at org.embulk.cli.EmbulkRun.run(EmbulkRun.java:90) [embulk-0.9.17:0.9.17]
 at org.embulk.cli.Main.main(Main.java:64) [embulk-0.9.17:0.9.17]
Caused by: java.net.SocketTimeoutException: Connect Timeout
 at org.eclipse.jetty.io.SelectorManager$ManagedSelector$ConnectTimeout.run(SelectorManager.java:966) ~[na:na]
 at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[na:1.8.0_252]
 at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_252]
 at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[na:1.8.0_252]
 at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[na:1.8.0_252]
 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_252]
 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_252]
 at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_252]
org.embulk.exec.PartialExecutionException: com.treasuredata.client.TDClientProcessingException: [EXECUTION_FAILURE] java.net.SocketTimeoutException: Connect Timeout The root cause: java.net.SocketTimeoutException: Connect Timeout
 at org.embulk.exec.BulkLoader$LoaderState.buildPartialExecuteException(BulkLoader.java:340)
 at org.embulk.exec.BulkLoader.doRun(BulkLoader.java:566)
 at org.embulk.exec.BulkLoader.access$000(BulkLoader.java:35)
 at org.embulk.exec.BulkLoader$1.run(BulkLoader.java:353)
 at org.embulk.exec.BulkLoader$1.run(BulkLoader.java:350)
 at org.embulk.spi.Exec.doWith(Exec.java:22)
 at org.embulk.exec.BulkLoader.run(BulkLoader.java:350)
 at org.embulk.EmbulkEmbed.run(EmbulkEmbed.java:178)
 at org.embulk.EmbulkRunner.runInternal(EmbulkRunner.java:292)
 at org.embulk.EmbulkRunner.run(EmbulkRunner.java:156)
 at org.embulk.cli.EmbulkRun.runSubcommand(EmbulkRun.java:433)
 at org.embulk.cli.EmbulkRun.run(EmbulkRun.java:90)
 at org.embulk.cli.Main.main(Main.java:64)
 Suppressed: java.lang.NullPointerException
  at org.embulk.exec.BulkLoader.doCleanup(BulkLoader.java:463)
  at org.embulk.exec.BulkLoader$3.run(BulkLoader.java:397)
  at org.embulk.exec.BulkLoader$3.run(BulkLoader.java:394)
  at org.embulk.spi.Exec.doWith(Exec.java:22)
  at org.embulk.exec.BulkLoader.cleanup(BulkLoader.java:394)
  at org.embulk.EmbulkEmbed.run(EmbulkEmbed.java:181)
  ... 5 more
Caused by: com.treasuredata.client.TDClientProcessingException: [EXECUTION_FAILURE] java.net.SocketTimeoutException: Connect Timeout The root cause: java.net.SocketTimeoutException: Connect Timeout
 at com.treasuredata.client.TDHttpClient.submitRequest(TDHttpClient.java:429)
 at com.treasuredata.client.TDHttpClient.call(TDHttpClient.java:628)
 at com.treasuredata.client.TDHttpClient.call(TDHttpClient.java:594)
 at com.treasuredata.client.TDClient.doGet(TDClient.java:193)
 at com.treasuredata.client.TDClient.jobStatus(TDClient.java:618)
 at org.embulk.input.td.TdInputPlugin.waitJobCompletion(TdInputPlugin.java:177)
 at org.embulk.input.td.TdInputPlugin.getTdJob(TdInputPlugin.java:155)
 at org.embulk.input.td.TdInputPlugin.transaction(TdInputPlugin.java:77)
 at org.embulk.exec.BulkLoader.doRun(BulkLoader.java:507)
 ... 11 more

Error: com.treasuredata.client.TDClientProcessingException: [EXECUTION_FAILURE] java.net.SocketTimeoutException: Connect Timeout The root cause: java.net.SocketTimeoutException: Connect Timeout
2020-07-15 23:43:20.443 +0000 [ERROR] (0137@[0:****-workflow]+****_transfer_periodically+main+****^sub+load+hadoop^sub+for-0=config=0=%7B%22destin^sub+****_dashboard^sub+for-0=output_c=0=%7B%22destin^sub+load^sub) io.digdag.core.agent.OperatorManager: Task failed with unexpected error: Command failed with code 1
java.lang.RuntimeException: Command failed with code 1
 at io.digdag.standards.operator.ShOperatorFactory$ShOperator.runTask(ShOperatorFactory.java:143)
 at io.digdag.util.BaseOperator.run(BaseOperator.java:35)
 at io.digdag.core.agent.OperatorManager.callExecutor(OperatorManager.java:315)
 at io.digdag.core.agent.OperatorManager.runWithWorkspace(OperatorManager.java:257)
 at io.digdag.core.agent.OperatorManager.lambda$runWithHeartbeat$2(OperatorManager.java:137)
 at io.digdag.core.agent.ExtractArchiveWorkspaceManager.withExtractedArchive(ExtractArchiveWorkspaceManager.java:77)
 at io.digdag.core.agent.OperatorManager.runWithHeartbeat(OperatorManager.java:135)
 at io.digdag.core.agent.OperatorManager.run(OperatorManager.java:119)
 at io.digdag.core.agent.MultiThreadAgent.lambda$null$0(MultiThreadAgent.java:127)
 at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
 at java.util.concurrent.FutureTask.run(FutureTask.java:266)
 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
 at java.lang.Thread.run(Thread.java:748)