Initial commit

.gitignore

+/build
+/dist
+/osqa_migrator.egg-info
+*.py[co]

osqa_migrator/__main__.py

+# This migration tool converts OSQA data to Askbot, optimized for big databases
+# Copyright (C) 2016  Pietro Albini <pietroalbini@ubuntu.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import print_function
+
+import sys
+
+from .migrator import Migrator
+
+
+def main():
+    args = sys.argv[1:]
+
+    migrator = Migrator(args[0], args[1], args[2])
+    migrator.migrate()

osqa_migrator/django_integration.py

+# This migration tool converts OSQA data to Askbot, optimized for big databases
+# Copyright (C) 2016  Pietro Albini <pietroalbini@ubuntu.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import os
+import sys
+import site
+import pkg_resources
+
+
+def load_django(path, env):
+    # Load the app's virtualenv
+    site_packages = os.path.join(
+        env, 'lib', 'python%s' % sys.version[:3], 'site-packages'
+    )
+    site.addsitedir(site_packages)
+
+    # Switch to the app's directory
+    app_path = os.path.abspath(os.path.expanduser(path))
+    sys.path.append(app_path)
+    os.chdir(app_path)
+
+    # Refresh pkg_resources working set to discover new modules
+    pkg_resources.working_set = pkg_resources.WorkingSet()
+
+    # Tell Django where to find settings
+    os.environ["DJANGO_SETTINGS_MODULE"] = "settings"
+
+    # Load django
+    import django
+    django.setup()
+
+    return django
+
+
+def load_models():
+    import askbot
+    return askbot.models
+
+
+def load_openid_models():
+    import askbot.deps.django_authopenid
+    return askbot.deps.django_authopenid.models

osqa_migrator/migrator.py

+# This migration tool converts OSQA data to Askbot, optimized for big databases
+# Copyright (C) 2016  Pietro Albini <pietroalbini@ubuntu.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+from . import django_integration
+from . import utils
+
+
+class Migrator:
+
+    def __init__(self, osqa_conn, askbot_path, askbot_env):
+        # Connect to the OSQA database
+        self.osqa = utils.Connection(osqa_conn)
+
+        # Load Askbot
+        self.django = django_integration.load_django(askbot_path, askbot_env)
+        self.models = django_integration.load_models()
+        self.openid_models = django_integration.load_openid_models()
+
+    def set_next_id(self, table, field, value):
+        """Set the next ID for a sequence"""
+        self.askbot_query(
+            "SELECT setval("
+                "(SELECT pg_get_serial_sequence(%s, %s)), %s"
+            ")"
+        ";", table, field, value)
+
+    def askbot_query(self, query, *params):
+        """Execute a query against Askbot"""
+        with self.django.db.connection.cursor() as cursor:
+            cursor.execute(query, params)
+            return cursor.fetchall()
+
+    def migrate(self):
+        """Start the migration process"""
+        self.migrate_users()
+        self.migrate_user_logins()
+
+    def migrate_users(self):
+        """Migrate users to Askbot"""
+        migrate_fields_user = [
+            "id", "email", "username", "first_name", "last_name", "last_login",
+            "date_joined",
+        ]
+        migrate_fields_profile = [
+            "website", "email_isvalid", "real_name", "date_of_birth",
+            "location", "last_seen",
+        ]
+
+        last_id = 0
+        tbl = "auth_user a, forum_user f WHERE a.id = f.user_ptr_id"
+        for user in self.osqa.table_items(tbl, what="users"):
+            last_id = user["id"]
+
+            # Check if the user already exists
+            try:
+                self.models.User.objects.get(id=user["id"])
+                continue
+            except self.models.User.DoesNotExist:
+                pass
+
+            # Create a new User object
+            new = self.models.User()
+
+            # Copy some fields
+            for field in migrate_fields_user:
+                setattr(new, field, user[field])
+
+            # Update some other details of the user
+            new.set_unusable_password()
+            if user["is_superuser"]:
+                new.set_admin_status()
+
+            # Save the user
+            new.save()
+
+            # Update the user profile
+            for field in migrate_fields_profile:
+                setattr(new, field, user[field])
+
+            # Update the about text
+            new.update_localized_profile(about=user["about"])
+
+            # Save the user again
+            new.save()
+
+        # Update the next ID value
+        self.set_next_id("auth_user", "id", last_id + 1)
+
+    def migrate_user_logins(self):
+        """Migrate the login table"""
+        last_id = 0
+        table = "forum_authkeyuserassociation"
+        for login in self.osqa.table_items(table, what="user logins"):
+            last_id = login["id"]
+
+            # Check if the association already exists
+            try:
+                self.openid_models.UserAssociation.objects.get(id=login["id"])
+                continue
+            except self.openid_models.UserAssociation.DoesNotExist:
+                pass
+
+            # Migrate login data
+            assoc = self.openid_models.UserAssociation()
+            assoc.id = login["id"]
+            assoc.openid_url = login["key"]
+            assoc.user_id = login["user_id"]
+            assoc.provider_name = login["provider"]
+            assoc.last_used_timestamp = login["added_at"]
+            try:
+                assoc.save()
+            except self.django.db.utils.IntegrityError:
+                # OSQA forgot to delete a record -- don't add it
+                pass
+
+        # Update the next ID value
+        self.set_next_id("django_authopenid_userassociation", "id", last_id + 1)

osqa_migrator/utils.py

+# This migration tool converts OSQA data to Askbot, optimized for big databases
+# Copyright (C) 2016  Pietro Albini <pietroalbini@ubuntu.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import sys
+
+import psycopg2
+
+
+class ProgressBar:
+
+    def __init__(self, action, items, total):
+        self.action = action
+        self.items = items
+        self.current = 0
+        self.total = total
+
+    def __enter__(self):
+        self.start()
+        return self
+
+    def __exit__(self, *_):
+        self.stop()
+
+    def start(self):
+        self.to(0)
+
+    def stop(self):
+        self.repaint()
+        print()
+
+    def to(self, current):
+        """Set the current value of the progress bar"""
+        self.current = current
+        self.repaint()
+
+    def repaint(self):
+        """Repaint the progress bar"""
+        if self.current != self.total:
+            msg = "%s: %s/%s" % (self.action, self.current, self.total)
+        else:
+            msg = "%s: completed (%s %s)" % (
+                    self.action, self.total, self.items
+            ),
+
+        sys.stdout.write("\r%s" % msg)
+        sys.stdout.flush()
+
+
+class Connection:
+
+    def __init__(self, connstring):
+        self.conn = psycopg2.connect(connstring)
+
+    def query(self, query, *params):
+        """Execute a query against the DB"""
+        cursor = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
+        cursor.execute(query, params)
+        return cursor.fetchall()
+
+    def table_items(self, name, batch=100, what=None):
+        """Return items in a table without loading all of them in memory"""
+        # This loads elements in batches, to avoid loading too much stuff in
+        # memory and going OOM
+        total = self.query("SELECT COUNT(*) FROM %s;" % name)[0][0]
+
+        pb = None
+        if what is not None:
+            pb = ProgressBar("Migrating %s" % what, what, total)
+            pb.start()
+
+        steps = total / batch
+        if total % batch != 0:
+            steps += 1
+
+        for i in range(steps):
+            query = (
+                "SELECT * FROM %s LIMIT %s OFFSET %s;" %
+                (name, batch, i * batch)
+            )
+            for (i2, item) in enumerate(self.query(query)):
+                if pb is not None:
+                    pb.to((i * batch) + i2 + 1)
+
+                yield item
+
+        if pb:
+            pb.stop()

setup.py

+#!/usr/bin/python3
+# This migration tool converts OSQA data to Askbot, optimized for big databases
+# Copyright (C) 2016  Pietro Albini <pietroalbini@ubuntu.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import setuptools
+
+
+setuptools.setup(
+    name = "osqa-migrator",
+    version = "1",
+    url = "http://code.ubuntu-it.org/ubuntu-it-ask/osqa-migrator",
+
+    license = "GPL-3+",
+
+    author = "Pietro Albini",
+    author_email = "pietroalbini@ubuntu.com",
+
+    description = "Migrate an OSQA database to an Askbot instance",
+
+    packages = [
+        "osqa_migrator",
+    ],
+
+    entry_points = {
+        "console_scripts": [
+            "osqa-migrator = osqa_migrator.__main__:main"
+        ]
+    },
+
+    install_requires = [
+        "psycopg2",
+    ],
+
+    zip_safe = False,
+
+    classifiers = [
+        "PyPI :: Ignore",
+    ],
+)