Merge pull request #34 from NicoHood/version15

Release 1.5

README.md

@@ -4,6 +4,7 @@
 
 Python script for building the Track and Playlist database for the newer gen IPod Shuffle.
 Forked from the [shuffle-db-ng project](https://code.google.com/p/shuffle-db-ng/)
+and improved my nims11 and NicoHood.
 
 Just put your audio files into the mass storage of your IPod and shuffle.py will do the rest.
 ```
@@ -13,7 +14,7 @@ usage: ipod-shuffle-4g.py [-h] [-t] [-p] [-u] [-g TRACK_GAIN]
                           path
 
 Python script for building the Track and Playlist database for the newer gen
-IPod Shuffle. Version 1.4
+IPod Shuffle. Version 1.5
 
 positional arguments:
   path                  Path to the IPod's root directory
@@ -50,7 +51,9 @@ optional arguments:
 #### Dependencies
 
 This script requires:
-* [Python 2.7](http://www.python.org/download/releases/2.7/)
+* [Python 3](https://www.python.org/download/releases/3.0/)
+
+Optional album/artist and auto-id3-playlists support:
 * [Mutagen](https://code.google.com/p/mutagen/)
 
 Optional Voiceover support
@@ -61,18 +64,18 @@ Optional Voiceover support
 
 ##### Ubuntu
 
-`apt-get install python-mutagen libttspico*`
+`apt-get install python3 python-mutagen libttspico*`
 
 ##### Arch Linux
 
-From the **Extra** repository: `pacman -S python2 mutagen` and optional `pacman -S espeak` or from the AUR: `svox-pico-bin` ([link](https://aur.archlinux.org/packages/svox-pico-bin/))
+From the **Extra** repository: `pacman -S python` and optional `pacman -S python-mutagen espeak` or from the AUR: `svox-pico-bin` ([link](https://aur.archlinux.org/packages/svox-pico-bin/))
 
 You can also [install the script from AUR](https://aur.archlinux.org/packages/ipod-shuffle-4g/).
 
 ##### Gentoo Linux
 
 ```bash
-PYTHON_TARGETS="python2_7" emerge -av media-libs/mutagen
+PYTHON_TARGETS="python3" emerge -av media-libs/mutagen
 layman --add=ikelos
 layman --overlays="https://raw.githubusercontent.com/ahippo/rhvoice-gentoo-overlay/master/repositories.xml" --fetch --add=ahippo-rhvoice-overlay
 ACCEPT_KEYWORDS="~amd64" emerge -av app-accessibility/svox app-accessibility/rhvoice
@@ -145,6 +148,10 @@ Original data can be found via [wayback machine](https://web.archive.org/web/201
 # Version History
 
 ```
+1.5 Release (09.06.2020)
+* Port Script to Python3
+* Mutagen support is now optional
+
 1.4 Release (27.08.2016)
 * Catch "no space left" error #30
 * Renamed --voiceover to --track-voiceover

ipod-shuffle-4g.py

@@ -1,12 +1,11 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
+
+# Builtin libraries
 import sys
 import struct
-import urllib
+import urllib.request, urllib.parse, urllib.error
 import os
 import hashlib
-import mutagen
-import binascii
 import subprocess
 import collections
 import errno
@@ -16,6 +15,12 @@ import re
 import tempfile
 import signal
 
+# External libraries
+try:
+    import mutagen
+except ImportError:
+    mutagen = None
+
 audio_ext = (".mp3", ".m4a", ".m4b", ".m4p", ".aa", ".wav")
 list_ext = (".pls", ".m3u")
 def make_dir_if_absent(path):
@@ -27,19 +32,19 @@ def make_dir_if_absent(path):
 
 def raises_unicode_error(str):
     try:
-        str.decode('utf-8').encode('latin-1')
+        str.encode('latin-1')
         return False
     except (UnicodeEncodeError, UnicodeDecodeError):
         return True
 
 def hash_error_unicode(item):
-    return "".join(["{0:02X}".format(ord(x)) for x in reversed(hashlib.md5(item).digest()[:8])])
+    item_bytes = item.encode('utf-8')
-    pass
+    return "".join(["{0:02X}".format(ord(x)) for x in reversed(hashlib.md5(item_bytes).hexdigest()[:8])])
 
 def validate_unicode(path):
     path_list = path.split('/')
     last_raise = False
-    for i in xrange(len(path_list)):
+    for i in range(len(path_list)):
         if raises_unicode_error(path_list[i]):
             path_list[i] = hash_error_unicode(path_list[i])
             last_raise = True
@@ -61,11 +66,11 @@ def splitpath(path):
     return path.split(os.sep)
 
 def get_relpath(path, basepath):
-    commonprefix = os.sep.join(os.path.commonprefix(map(splitpath, [path, basepath])))
+    commonprefix = os.sep.join(os.path.commonprefix(list(map(splitpath, [path, basepath]))))
     return os.path.relpath(path, commonprefix)
 
 def is_path_prefix(prefix, path):
-    return prefix == os.sep.join(os.path.commonprefix(map(splitpath, [prefix, path])))
+    return prefix == os.sep.join(os.path.commonprefix(list(map(splitpath, [prefix, path]))))
 
 def group_tracks_by_id3_template(tracks, template):
     grouped_tracks_dict = {}
@@ -101,21 +106,21 @@ class Text2Speech(object):
         # Check for pico2wave voiceover
         if not exec_exists_in_path("pico2wave"):
             Text2Speech.valid_tts['pico2wave'] = False
-            print "Warning: pico2wave not found, voicever won't be generated using it."
+            print("Warning: pico2wave not found, voicever won't be generated using it.")
         else:
             voiceoverAvailable = True
 
         # Check for espeak voiceover
         if not exec_exists_in_path("espeak"):
             Text2Speech.valid_tts['espeak'] = False
-            print "Warning: espeak not found, voicever won't be generated using it."
+            print("Warning: espeak not found, voicever won't be generated using it.")
         else:
             voiceoverAvailable = True
 
         # Check for Russian RHVoice voiceover
         if not exec_exists_in_path("RHVoice"):
             Text2Speech.valid_tts['RHVoice'] = False
-            print "Warning: RHVoice not found, Russian voicever won't be generated."
+            print("Warning: RHVoice not found, Russian voicever won't be generated.")
         else:
             voiceoverAvailable = True
 
@@ -132,8 +137,8 @@ class Text2Speech(object):
             return True
 
         # ensure we deal with unicode later
-        if not isinstance(text, unicode):
+        if not isinstance(text, str):
-            text = unicode(text, 'utf-8')
+            text = str(text, 'utf-8')
         lang = Text2Speech.guess_lang(text)
         if lang == "ru-RU":
             return Text2Speech.rhvoice(out_wav_path, text)
@@ -149,7 +154,7 @@ class Text2Speech(object):
     @staticmethod
     def guess_lang(unicodetext):
         lang = 'en-GB'
-        if re.search(u"[А-Яа-я]", unicodetext) is not None:
+        if re.search("[А-Яа-я]", unicodetext) is not None:
             lang = 'ru-RU'
         return lang
 
@@ -196,7 +201,7 @@ class Record(object):
         self.trackgain = parent.trackgain
 
     def __getitem__(self, item):
-        if item not in self._struct.keys():
+        if item not in list(self._struct.keys()):
             raise KeyError
         return self._fields.get(item, self._struct[item][1])
 
@@ -204,18 +209,16 @@ class Record(object):
         self._fields[item] = value
 
     def construct(self):
-        output = ""
+        output = bytes()
-        for i in self._struct.keys():
+        for i in list(self._struct.keys()):
             (fmt, default) = self._struct[i]
-            if fmt == "4s":
-                fmt, default = "I", int(binascii.hexlify(default), 16)
             output += struct.pack("<" + fmt, self._fields.get(i, default))
         return output
 
     def text_to_speech(self, text, dbid, playlist = False):
         if self.track_voiceover and not playlist or self.playlist_voiceover and playlist:
             # Create the voiceover wav file
-            fn = "".join(["{0:02X}".format(ord(x)) for x in reversed(dbid)])
+            fn = ''.join(format(x, '02x') for x in reversed(dbid))
             path = os.path.join(self.base, "iPod_Control", "Speakable", "Tracks" if not playlist else "Playlists", fn + ".wav")
             return Text2Speech.text2speech(path, text)
         return False
@@ -265,7 +268,7 @@ class TunesSD(Record):
         self.track_header = TrackHeader(self)
         self.play_header = PlaylistHeader(self)
         self._struct = collections.OrderedDict([
-                           ("header_id", ("4s", "shdb")),
+                           ("header_id", ("4s", b"bdhs")), # shdb
                            ("unknown1", ("I", 0x02000003)),
                            ("total_length", ("I", 64)),
                            ("total_number_of_tracks", ("I", 0)),
@@ -277,7 +280,7 @@ class TunesSD(Record):
                            ("total_tracks_without_podcasts", ("I", 0)),
                            ("track_header_offset", ("I", 64)),
                            ("playlist_header_offset", ("I", 0)),
-                           ("unknown4", ("20s", "\x00" * 20)),
+                           ("unknown4", ("20s", b"\x00" * 20)),
                                                ])
 
     def construct(self):
@@ -302,7 +305,7 @@ class TrackHeader(Record):
         self.base_offset = 0
         Record.__init__(self, parent)
         self._struct = collections.OrderedDict([
-                           ("header_id", ("4s", "shth")),
+                           ("header_id", ("4s", b"hths")), # shth
                            ("total_length", ("I", 0)),
                            ("number_of_tracks", ("I", 0)),
                            ("unknown1", ("Q", 0)),
@@ -314,7 +317,7 @@ class TrackHeader(Record):
         output = Record.construct(self)
 
         # Construct the underlying tracks
-        track_chunk = ""
+        track_chunk = bytes()
         for i in self.tracks:
             track = Track(self)
             verboseprint("[*] Adding track", i)
@@ -328,13 +331,13 @@ class Track(Record):
     def __init__(self, parent):
         Record.__init__(self, parent)
         self._struct = collections.OrderedDict([
-                           ("header_id", ("4s", "shtr")),
+                           ("header_id", ("4s", b"rths")), # shtr
                            ("header_length", ("I", 0x174)),
                            ("start_at_pos_ms", ("I", 0)),
                            ("stop_at_pos_ms", ("I", 0)),
                            ("volume_gain", ("I", int(self.trackgain))),
                            ("filetype", ("I", 1)),
-                           ("filename", ("256s", "\x00" * 256)),
+                           ("filename", ("256s", b"\x00" * 256)),
                            ("bookmark", ("I", 0)),
                            ("dontskip", ("B", 1)),
                            ("remember", ("B", 0)),
@@ -352,33 +355,36 @@ class Track(Record):
                            ("unknown4", ("Q", 0)),
                            ("dbid", ("8s", 0)),
                            ("artistid", ("I", 0)),
-                           ("unknown5", ("32s", "\x00" * 32)),
+                           ("unknown5", ("32s", b"\x00" * 32)),
                            ])
 
     def populate(self, filename):
-        self["filename"] = self.path_to_ipod(filename)
+        self["filename"] = self.path_to_ipod(filename).encode('utf-8')
 
         if os.path.splitext(filename)[1].lower() in (".m4a", ".m4b", ".m4p", ".aa"):
             self["filetype"] = 2
 
         text = os.path.splitext(os.path.basename(filename))[0]
+
+        # Try to get album and artist information with mutagen
+        if mutagen:
             audio = None
             try:
                 audio = mutagen.File(filename, easy = True)
             except:
-            print "Error calling mutagen. Possible invalid filename/ID3Tags (hyphen in filename?)"
+                print("Error calling mutagen. Possible invalid filename/ID3Tags (hyphen in filename?)")
             if audio:
                 # Note: Rythmbox IPod plugin sets this value always 0.
                 self["stop_at_pos_ms"] = int(audio.info.length * 1000)
 
-            artist = audio.get("artist", [u"Unknown"])[0]
+                artist = audio.get("artist", ["Unknown"])[0]
                 if artist in self.artists:
                     self["artistid"] = self.artists.index(artist)
                 else:
                     self["artistid"] = len(self.artists)
                     self.artists.append(artist)
 
-            album = audio.get("album", [u"Unknown"])[0]
+                album = audio.get("album", ["Unknown"])[0]
                 if album in self.albums:
                     self["albumid"] = self.albums.index(album)
                 else:
@@ -386,12 +392,12 @@ class Track(Record):
                     self.albums.append(album)
 
                 if audio.get("title", "") and audio.get("artist", ""):
-                text = u" - ".join(audio.get("title", u"") + audio.get("artist", u""))
+                    text = " - ".join(audio.get("title", "") + audio.get("artist", ""))
 
         # Handle the VoiceOverData
-        if isinstance(text, unicode):
+        if isinstance(text, str):
             text = text.encode('utf-8', 'ignore')
-        self["dbid"] = hashlib.md5(text).digest()[:8] #pylint: disable-msg=E1101
+        self["dbid"] = hashlib.md5(text).digest()[:8]
         self.text_to_speech(text, self["dbid"])
 
 class PlaylistHeader(Record):
@@ -399,16 +405,16 @@ class PlaylistHeader(Record):
         self.base_offset = 0
         Record.__init__(self, parent)
         self._struct = collections.OrderedDict([
-                          ("header_id", ("4s", "shph")),
+                          ("header_id", ("4s", b"hphs")), #shph
                           ("total_length", ("I", 0)),
                           ("number_of_playlists", ("I", 0)),
-                          ("number_of_non_podcast_lists", ("2s", "\xFF\xFF")),
+                          ("number_of_non_podcast_lists", ("2s", b"\xFF\xFF")),
-                          ("number_of_master_lists", ("2s", "\x01\x00")),
+                          ("number_of_master_lists", ("2s", b"\x01\x00")),
-                          ("number_of_non_audiobook_lists", ("2s", "\xFF\xFF")),
+                          ("number_of_non_audiobook_lists", ("2s", b"\xFF\xFF")),
-                          ("unknown2", ("2s", "\x00" * 2)),
+                          ("unknown2", ("2s", b"\x00" * 2)),
                                               ])
 
-    def construct(self, tracks): #pylint: disable-msg=W0221
+    def construct(self, tracks):
         # Build the master list
         masterlist = Playlist(self)
         verboseprint("[+] Adding master playlist")
@@ -426,7 +432,7 @@ class PlaylistHeader(Record):
                 playlistcount += 1
                 chunks += [construction]
             else:
-                print "Error: Playlist does not contain a single track. Skipping playlist."
+                print("Error: Playlist does not contain a single track. Skipping playlist.")
 
         self["number_of_playlists"] = playlistcount
         self["total_length"] = 0x14 + (self["number_of_playlists"] * 4)
@@ -439,27 +445,27 @@ class PlaylistHeader(Record):
             output += struct.pack("I", offset)
             offset += len(chunks[i])
 
-        return output + "".join(chunks)
+        return output + b"".join(chunks)
 
 class Playlist(Record):
     def __init__(self, parent):
         self.listtracks = []
         Record.__init__(self, parent)
         self._struct = collections.OrderedDict([
-                          ("header_id", ("4s", "shpl")),
+                          ("header_id", ("4s", b"lphs")), # shpl
                           ("total_length", ("I", 0)),
                           ("number_of_songs", ("I", 0)),
                           ("number_of_nonaudio", ("I", 0)),
-                          ("dbid", ("8s", "\x00" * 8)),
+                          ("dbid", ("8s", b"\x00" * 8)),
                           ("listtype", ("I", 2)),
-                          ("unknown1", ("16s", "\x00" * 16))
+                          ("unknown1", ("16s", b"\x00" * 16))
                                               ])
 
     def set_master(self, tracks):
         # By default use "All Songs" builtin voiceover (dbid all zero)
         # Else generate alternative "All Songs" to fit the speaker voice of other playlists
         if self.playlist_voiceover and (Text2Speech.valid_tts['pico2wave'] or Text2Speech.valid_tts['espeak']):
-            self["dbid"] = hashlib.md5("masterlist").digest()[:8] #pylint: disable-msg=E1101
+            self["dbid"] = hashlib.md5(b"masterlist").digest()[:8]
             self.text_to_speech("All songs", self["dbid"], True)
         self["listtype"] = 1
         self.listtracks = tracks
@@ -480,7 +486,7 @@ class Playlist(Record):
             dataarr = i.strip().split("=", 1)
             if dataarr[0].lower().startswith("file"):
                 num = int(dataarr[0][4:])
-                filename = urllib.unquote(dataarr[1]).strip()
+                filename = urllib.parse.unquote(dataarr[1]).strip()
                 if filename.lower().startswith('file://'):
                     filename = filename[7:]
                 if self.rename:
@@ -545,16 +551,14 @@ class Playlist(Record):
                 text = os.path.splitext(os.path.basename(filename))[0]
 
         # Handle the VoiceOverData
-        if isinstance(text, unicode):
+        self["dbid"] = hashlib.md5(text.encode('utf-8')).digest()[:8]
-            text = text.encode('utf-8', 'ignore')
-        self["dbid"] = hashlib.md5(text).digest()[:8] #pylint: disable-msg=E1101
         self.text_to_speech(text, self["dbid"], True)
 
-    def construct(self, tracks): #pylint: disable-msg=W0221
+    def construct(self, tracks):
         self["total_length"] = 44 + (4 * len(self.listtracks))
         self["number_of_songs"] = 0
 
-        chunks = ""
+        chunks = bytes()
         for i in self.listtracks:
             path = self.ipod_to_path(i)
             position = -1
@@ -563,8 +567,8 @@ class Playlist(Record):
             except:
                 # Print an error if no track was found.
                 # Empty playlists are handeled in the PlaylistHeader class.
-                print "Error: Could not find track \"" + path + "\"."
+                print("Error: Could not find track \"" + path + "\".")
-                print "Maybe its an invalid FAT filesystem name. Please fix your playlist. Skipping track."
+                print("Maybe its an invalid FAT filesystem name. Please fix your playlist. Skipping track.")
             if position > -1:
                 chunks += struct.pack("I", position)
                 self["number_of_songs"] += 1
@@ -596,11 +600,11 @@ class Shuffler(object):
           make_dir_if_absent(os.path.join(self.path, dirname))
 
     def dump_state(self):
-        print "Shuffle DB state"
+        print("Shuffle DB state")
-        print "Tracks", self.tracks
+        print("Tracks", self.tracks)
-        print "Albums", self.albums
+        print("Albums", self.albums)
-        print "Artists", self.artists
+        print("Artists", self.artists)
-        print "Playlists", self.lists
+        print("Playlists", self.lists)
 
     def populate(self):
         self.tunessd = TunesSD(self)
@@ -627,22 +631,28 @@ class Shuffler(object):
                     self.lists.append(os.path.abspath(dirpath))
 
         if self.auto_id3_playlists != None:
+            if mutagen:
                 for grouped_list in group_tracks_by_id3_template(self.tracks, self.auto_id3_playlists):
                     self.lists.append(grouped_list)
+            else:
+                print("Error: No mutagen found. Cannot generate auto-id3-playlists.")
+                sys.exit(1)
 
     def write_database(self):
+        print("Writing database. This may take a while...")
         with open(os.path.join(self.path, "iPod_Control", "iTunes", "iTunesSD"), "wb") as f:
             try:
                 f.write(self.tunessd.construct())
             except IOError as e:
-                print "I/O error({0}): {1}".format(e.errno, e.strerror)
+                print("I/O error({0}): {1}".format(e.errno, e.strerror))
-                print "Error: Writing iPod database failed."
+                print("Error: Writing iPod database failed.")
                 sys.exit(1)
-        print "Database written successfully:"
+
-        print "Tracks", len(self.tracks)
+        print("Database written successfully:")
-        print "Albums", len(self.albums)
+        print("Tracks", len(self.tracks))
-        print "Artists", len(self.artists)
+        print("Albums", len(self.albums))
-        print "Playlists", len(self.lists)
+        print("Artists", len(self.artists))
+        print("Playlists", len(self.lists))
 
 #
 # Read all files from the directory
@@ -661,7 +671,7 @@ def check_unicode(path):
                 if raises_unicode_error(item):
                     src = os.path.join(path, item)
                     dest = os.path.join(path, hash_error_unicode(item)) + os.path.splitext(item)[1].lower()
-                    print 'Renaming %s -> %s' % (src, dest)
+                    print('Renaming %s -> %s' % (src, dest))
                     os.rename(src, dest)
         else:
             ret_flag = (check_unicode(os.path.join(path, item)) or ret_flag)
@@ -669,7 +679,7 @@ def check_unicode(path):
                 src = os.path.join(path, item)
                 new_name = hash_error_unicode(item)
                 dest = os.path.join(path, new_name)
-                print 'Renaming %s -> %s' % (src, dest)
+                print('Renaming %s -> %s' % (src, dest))
                 os.rename(src, dest)
     return ret_flag
 
@@ -685,15 +695,15 @@ def nonnegative_int(string):
 
 def checkPathValidity(path):
     if not os.path.isdir(result.path):
-        print "Error finding IPod directory. Maybe it is not connected or mounted?"
+        print("Error finding IPod directory. Maybe it is not connected or mounted?")
         sys.exit(1)
 
     if not os.access(result.path, os.W_OK):
-        print 'Unable to get write permissions in the IPod directory'
+        print('Unable to get write permissions in the IPod directory')
         sys.exit(1)
 
 def handle_interrupt(signal, frame):
-    print "Interrupt detected, exiting..."
+    print("Interrupt detected, exiting...")
     sys.exit(1)
 
 if __name__ == '__main__':
@@ -701,7 +711,7 @@ if __name__ == '__main__':
 
     parser = argparse.ArgumentParser(description=
     'Python script for building the Track and Playlist database '
-    'for the newer gen IPod Shuffle. Version 1.4')
+    'for the newer gen IPod Shuffle. Version 1.5')
 
     parser.add_argument('-t', '--track-voiceover', action='store_true',
     help='Enable track voiceover feature')
@@ -738,34 +748,33 @@ if __name__ == '__main__':
     result = parser.parse_args()
 
     # Enable verbose printing if desired
-    # Smaller version for python3 available.
+    verboseprint = print if result.verbose else lambda *a, **k: None
-    # See https://stackoverflow.com/questions/5980042/how-to-implement-the-verbose-or-v-option-into-a-script
-    if result.verbose:
-        def verboseprint(*args):
-            # Print each argument separately so caller doesn't need to
-            # stuff everything to be printed into a single string
-            for arg in args:
-               print arg,
-            print
-    else:
-        verboseprint = lambda *a: None      # do-nothing function
 
     checkPathValidity(result.path)
 
     if result.rename_unicode:
         check_unicode(result.path)
 
+    if not mutagen:
+        print("Warning: No mutagen found. Database will not contain any album nor artist information.")
+
     verboseprint("Playlist voiceover requested:", result.playlist_voiceover)
     verboseprint("Track voiceover requested:", result.track_voiceover)
     if (result.track_voiceover or result.playlist_voiceover):
         if not Text2Speech.check_support():
-            print "Error: Did not find any voiceover program. Voiceover disabled."
+            print("Error: Did not find any voiceover program. Voiceover disabled.")
             result.track_voiceover = False
             result.playlist_voiceover = False
         else:
             verboseprint("Voiceover available.")
 
-    shuffle = Shuffler(result.path, track_voiceover=result.track_voiceover, playlist_voiceover=result.playlist_voiceover, rename=result.rename_unicode, trackgain=result.track_gain, auto_dir_playlists=result.auto_dir_playlists, auto_id3_playlists=result.auto_id3_playlists)
+    shuffle = Shuffler(result.path,
+                       track_voiceover=result.track_voiceover,
+                       playlist_voiceover=result.playlist_voiceover,
+                       rename=result.rename_unicode,
+                       trackgain=result.track_gain,
+                       auto_dir_playlists=result.auto_dir_playlists,
+                       auto_id3_playlists=result.auto_id3_playlists)
     shuffle.initialize()
     shuffle.populate()
     shuffle.write_database()