-
Notifications
You must be signed in to change notification settings - Fork 0
/
bucket_migration.py
108 lines (73 loc) · 3.95 KB
/
bucket_migration.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
# -*- coding: utf-8 -*-
"""bucket_migration.ipynb
Automatically generated by Colaboratory.
Original file is located at
https://colab.research.google.com/drive/1dNG0IY9fLtiLTpz-ELjwAuDWO59koS-S
#### Login using the gcloud CLI
"""
!gcloud auth login
"""#### Set the project that contains your bucket"""
# !gcloud config set project project-id-1
!gcloud config set project project-id-2
"""#### CREATE A SERVICE ACCOUNT | just in case you dont have one"""
# !gcloud iam service-accounts create readonly-data-viewer --description="Service account for read-only access to Datastore" --display-name="Datastore ReadOnly"
"""#### Set the service account permissions"""
# !gcloud projects add-iam-policy-binding project-id-2 --member=serviceAccount:readonly-data-viewer@project-id-2.iam.gserviceaccount.com --role=roles/storage.objectViewer
# !gcloud projects add-iam-policy-binding project-id-2 --member=serviceAccount:readonly-data-viewer@project-id-2.iam.gserviceaccount.com --role=roles/storage.objectCreator
"""#### Get the keys"""
# !gcloud iam service-accounts keys create ./datastore-readonly3-key.json --iam-account readonly-data-viewer@project-id-2.iam.gserviceaccount.com
"""#### Once you conclude the work, remove risky permissions"""
# !gcloud projects remove-iam-policy-binding steam-project-id-2 --member=serviceAccount:readonly-data-viewer@steam-project-id-2.iam.gserviceaccount.com --role=roles/storage.objectCreator
"""#### Let's begin the game"""
import tempfile
from glob import glob
from google.cloud import storage
"""#### The functions I created for this task
"""
def get_files(bucket_name):
blobs = storage_client.list_blobs(bucket_name) ### Bucket initialization
files = [blob.name for blob in blobs] ### Get list of blobs in bucket
return files
def download_tmp_file(storage_client, bucket_name, blob_source, namef=''):
# Download files from bucket
bucket = storage_client.bucket(bucket_name) # Initialize bucket
blob = bucket.blob(blob_source) # Initialize blob
if namef!="":temp_local_path=f"/tmp/{namef}"
else:_, temp_local_path = tempfile.mkstemp() # Create a temporary file
print(temp_local_path)
folder = "/".join(temp_local_path.split("/")[:-1])
os.makedirs(folder, exist_ok=True)
blob.download_to_filename(temp_local_path) # Download blob to temp file
return temp_local_path
def upload_file(bucket_name, local_file, dest_buck_file):
bucket = storage_client.bucket(bucket_name) ### bucket initialize
blob = bucket.blob(dest_buck_file) ### destiny directory in the bucket
blob.upload_from_filename(local_file) ### local filename]
"""#### Set the location of your keys"""
import os
# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/content/datastore-readonly2-key.json" ### keys from account 1
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/content/datastore-readonly3-key.json" ### keys from account 2
"""#### Initialize your bucket"""
storage_client = storage.Client()
# bucket_name = 'bucket_name_1'
bucket_name = 'bucket_name_2'
blobs = storage_client.list_blobs(bucket_name)
"""#### Get the old bucket files into a list, remove those files you don't want to migrate"""
L = get_files(bucket_name=bucket_name)
L
"""#### Download the files into the temp folder in the Google Colab session"""
for filee in L:
download_tmp_file(storage_client=storage_client,
bucket_name=bucket_name,
blob_source=filee,
namef=f'bucket/{filee}'
)
"""#### Finally, upload those files into the new bucket. Remember to set the required permissions to the service account."""
for ff in glob("/tmp/bucket/**/*.*", recursive=True):
# print(ff)
dest = ff.replace("/tmp/bucket/", "")
print(ff, dest)
upload_file(bucket_name=bucket_name,
local_file=ff,
dest_buck_file=dest)
"""#### And that's it. Now you're a professional Google Cloud Storage migrator!"""