insert_sample_data

#!/usr/bin/env bash

set -e

MAPD_TCP_PORT=${MAPD_TCP_PORT:=6274}

DATA_URL=${DATA_URL:="https://data.mapd.com"}

ALLOW_DOWNLOADS=${ALLOW_DOWNLOADS:=true}

if hash wget 2>/dev/null; then
  GETTER="wget --continue"
elif hash curl 2>/dev/null; then
  GETTER="curl --continue - --remote-name --location"
else
  GETTER="echo Please download: "
fi

download_and_extract_file() {
  pushd $SAMPLE_PATH
  echo "- downloading and extracting $1"
  $GETTER "$DATA_URL/$1"
  tar xvf "$1"
  popd
}

while (( $# )); do
  case "$1" in
    --port)
      shift
      MAPD_TCP_PORT=$1 ;;
    --url)
      shift
      DATA_URL=$1 ;;
    --path)
      shift
      SAMPLE_PATH=$1 ;;
    --no-download)
      ALLOW_DOWNLOADS=false
      ;;
    --data)
      shift
      MAPD_DATA=$1
      ;;
    *)
      break ;;
  esac
  shift
done

if [ -z "${MAPD_DATA}" ]; then
  MAPD_DATA=${MAPD_DATA:="$PWD/storage"}
  echo "Using default storage directory: \"$MAPD_DATA\" if file path is not whitelisted use '--data /path_to_server_data_directory'"
fi

SAMPLE_PATH=${SAMPLE_PATH:="$MAPD_DATA/import/sample_datasets"}

MKRES=$(mkdir -p "$SAMPLE_PATH")
if ! mkdir -p "$SAMPLE_PATH" || [ ! -w "$SAMPLE_PATH" ] ; then
  SAMPLE_PATH2=$(mktemp -d)
  echo "Cannot write sample data to: $SAMPLE_PATH"
  echo "Saving instead to: $SAMPLE_PATH2"
  echo
  SAMPLE_PATH=$SAMPLE_PATH2
  mkdir -p "$SAMPLE_PATH"
fi

if [ "$ALLOW_DOWNLOADS" = false ] ; then
  GETTER="echo Using: "
fi

if [ "$ALLOW_DOWNLOADS" = true ] ; then
  pushd "$SAMPLE_PATH"
  rm -f manifest.tsv
  $GETTER "$DATA_URL/manifest.tsv"
  popd
fi

counter=1
while IFS=$'\t' read -r name size tablename filename ; do
  names[$counter]=$name
  sizes[$counter]=$size
  tables[$counter]=$tablename
  files[$counter]=$filename
  counter=$((counter+1))
done < "$SAMPLE_PATH/manifest.tsv"

echo "Enter dataset number to download, or 'q' to quit:"
table=" # | Dataset | Rows | Table Name | File Name"
for key in "${!files[@]}"; do
  table="$table
 $key) | ${names[$key]} | ${sizes[$key]} | ${tables[$key]} | ${files[$key]}"
done

column -t -s'|' <(echo "$table")

read -r idxs

if [ -z "$idxs" ]; then
  idxs=(${!files[@]})
fi

for idx in $idxs; do
  if [ "${files[$idx]}" ]; then
    filename="${files[$idx]}"
    download_and_extract_file "$filename"

    filebase="${filename%%.*}"

    echo "- adding schema"
    ./bin/heavysql heavyai -u admin -p HyperInteractive --port "$MAPD_TCP_PORT" < "$SAMPLE_PATH/$filebase"/*.sql
    table=${tables[$idx]}
    for csv in $SAMPLE_PATH/$filebase/*csv; do
      echo "- inserting file: $csv"
      echo "copy $table from '${csv}' with (quoted='true');" | ./bin/heavysql heavyai -u admin -p HyperInteractive --port "$MAPD_TCP_PORT"
    done
  fi
done