Completed post (image) adding feature. No duplicated images are loaded

igmirror.py

@@ -1,4 +1,5 @@
 #!/usr/bin/python3
+import threading
 import requests
 import hashlib
 import string
@@ -21,10 +22,18 @@ def add_igaccount(acc_id):
     if not os.path.exists(accfile):
         # get all profile data from instagram acc
         data = getig_user_data(acc_id)
+
+        # this tells us the user probably don't exist (or also might be a network error?)
         if len(data.keys()) == 0:
             print('E| User "{}" does not exist on Instagram'.format(acc_id))
             return 2
 
+        # we don't and can't mirror private profiles
+        #  (well, we can if we follow them and follow back, but we just don't need that)
+        if data['graphql']['user']['is_private']:
+            print('E| User "{}" is a private account. We just won\'t mirror that!'.format(acc_id))
+            return 3
+
         # get account display name to create it
         name = data['graphql']['user']['full_name']
         name = re.sub(r'[^a-zA-Z0-9_\s]', '', name)
@@ -40,17 +49,27 @@ def add_igaccount(acc_id):
 
         # save the account login information for updates and mirroring
         db_set('accounts', acc_id, account)
+        db_set('posts', acc_id, [])
 
         # set Pixelfed account data for the username
         pixelfed_setpic(acc_id, data['graphql']['user']['profile_pic_url'])
         pixelfed_setinfo(acc_id, data['graphql']['user']['biography'],\
             data['graphql']['user']['external_url'])
+        threading.Thread(target=pixelfed_dlposts, args=(acc_id, data['graphql']['user'])).start()
     else:
         print('W| User "{}" already exists in local database'.format(acc_id))
     return 0
 
 def update_igaccount(acc_id):
-    print('update igaccount')
+    # if account does not exist, we stop the mirroring process
+    accfile = './db/accounts/{}'.format(acc_id)
+    if not os.path.exists(accfile):
+        print('E| User "'+acc_id+'" has not been created yet, maybe you wanted to call /<username>/add ?')
+        return 1
+
+    # do it on a thread because it might take long to download the latest posts
+    data = getig_user_data(acc_id)
+    threading.Thread(target=pixelfed_dlposts, args=(acc_id, data['graphql']['user'])).start()
 
 def update_allaccounts():
     print('update all accounts')
@@ -88,19 +107,115 @@ def pixelfed_token_url(url='', _cookies=None):
     _token = re.search(r'name="_token".+value="([^"]+)"', r.text).group(1)
     return r.cookies, _token
 
+def pixelfed_dlposts(acc_id, data):
+    ts = []
+    for edge in data['edge_owner_to_timeline_media']['edges']:
+        ts.append(edge['node']['taken_at_timestamp'])
+    for edge in data['edge_felix_video_timeline']['edges']:
+        ts.append(edge['node']['taken_at_timestamp'])
+    ts = sorted(ts)
+    items = []
+    for t in ts:
+        brkit = False
+        for edge in data['edge_owner_to_timeline_media']['edges']:
+            if edge['node']['taken_at_timestamp'] == t:
+                items.append(edge['node'])
+                brkit = True
+                break
+        if brkit:
+            continue
+        for edge in data['edge_felix_video_timeline']['edges']:
+            if edge['node']['taken_at_timestamp'] == t:
+                items.append(edge['node'])
+                break
+
+    # mirror posts from the account (only the last N, without loading more),
+    #   but only the ones that has not already been imported
+    accposts = db_get('posts', acc_id)
+    accdata = db_get('accounts', acc_id)
+    for item in items:
+        if item['is_video']:
+            continue
+
+        if item['shortcode'] in accposts:
+            print('I| skipping IG post {}:{}. Already added'.format(acc_id, item['shortcode']))
+            continue
+
+        print('I| processing IG post {}:{}'.format(acc_id, item['shortcode']))
+        ig_url = 'https://www.instagram.com/p/{}/'.format(item['shortcode'])
+        title = item['title'] if 'title' in item else None
+        media = None
+        if not item['is_video']:
+            media = item['display_url']
+        else:
+            media = item['video_url']
+        caption = item['edge_media_to_caption']['edges'][0]['node']['text'] \
+            if len(item['edge_media_to_caption']['edges']) > 0 else ''
+        altcaption = item['accessibility_caption'] if 'accessibility_caption' in item else None
+
+        # for now, we only support images (not videos :( )
+        if not item['is_video']:
+            _token, jsdata = pixelfed_postimage(acc_id, media, accdata)
+            if not jsdata:
+                print('E| Could not upload media for {}:{}'.format(acc_id, item['shortcode']))
+                continue
+            jsdata['description'] = ig_url
+            caption = caption[0:136]+'...' if len(caption) > 140 else caption
+            jsdata['alt'] = altcaption[0:136]+'...' if len(altcaption) > 140 else altcaption
+            jsdata['cw'] = False
+            print(jsdata)
+            r = requests.post('https://'+config()['instance']+'/api/compose/v0/publish',\
+                json={"media": [jsdata], "caption": caption, "visibility": "public", "cw": False,\
+                    "comments_disabled": False, "place": False, "tagged": [],"optimize_media": True},\
+                cookies=accdata['cookie'],\
+                headers={
+                    'Content-Type': 'application/json',
+                    'X-CSRF-TOKEN': _token,
+                    'X-Requested-With': 'XMLHttpRequest',
+                    'X-XSRF-TOKEN': accdata['cookie']['XSRF-TOKEN']
+                }
+            )
+            accposts.append(item['shortcode'])
+            print('I| uploaded media for {}:{} : {}'.format(acc_id, item['shortcode'], r.status_code))
+
+    print('I| done uploading media for {}'.format(acc_id))
+    db_set('posts', acc_id, accposts)
+
+# upload media and return data
+def pixelfed_postimage(acc_id, image_url, accdata=None):
+    if accdata is None:
+        accdata = db_get('accounts', acc_id)
+    cachef = pixelfed_cacheimg(image_url)
+    _, _token = pixelfed_token_url('', accdata['cookie'])
+    r = requests.post( 'https://'+config()['instance']+'/api/compose/v0/media/upload',\
+        files={'file': open(cachef, 'rb')}, cookies=accdata['cookie'],\
+        headers={
+            'X-CSRF-TOKEN': _token,
+            'X-Requested-With': 'XMLHttpRequest',
+            'X-XSRF-TOKEN': accdata['cookie']['XSRF-TOKEN']
+        }
+    )
+    if r.status_code == 200:
+        return _token, json.loads(r.text)
+    return None, False
+
+
+# get the image by URL but cache it forever, as if the profile changes the pic
+#  the url will be different, and therefore, the sum will also be different
+def pixelfed_cacheimg(image_url):
+    cachef = './cache/{}.jpg'.format(md5sum(image_url))
+    if not os.path.exists(cachef):
+        r = requests.get(image_url)
+        w = open(cachef, 'wb')
+        w.write(r.content)
+        w.close()
+    return cachef
+
 def pixelfed_setpic(acc_id, pic_url, count=0):
     count += 1
     pixelfed_login(acc_id)
 
-    # get the image by URL but cache it forever, as if the profile changes the pic
+    cachef = pixelfed_cacheimg(pic_url)
-    #  the url will be different, and therefore, the sum will also be different
-    cachef = './cache/{}.jpg'.format(md5sum(pic_url))
-    if not os.path.exists(cachef):
-        r = requests.get(pic_url)
-        w = open(cachef, 'wb')
-        w.write(r.content)
-        w.close()
-
     accdata = db_get('accounts', acc_id)
     _, _token = pixelfed_token_url('/settings/home', accdata['cookie'])
     r = requests.post( 'https://'+config()['instance']+'/settings/avatar',\