Initial commit

LICENSE

+Copyright (c) 2018 Pietro Albini <pietroalbini@ubuntu-it.org>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

+# Archivio di chiedi.ubuntu-it.org
+
+Questo repository contiene l'archivio di Chiedi, il portale di domande e
+risposte di Ubuntu-it. Il codice contenuto è rilasciato sotto licenza MIT,
+mentre i dati sono rilasciati sotto licenza CreativeCommons BY-SA 3.0.
+
+I dati sono contenuti nella directory `data/`.
+
+## Estrazione dei dati
+
+Se si dispone dell'accesso al database di produzione di Chiedi è possibile
+estrarre i dati e salvarli nel repository. Per farlo è necessario avere
+installati i pacchetti `python3` e `python3-psycopg2`, poi si può eseguire:
+
+```
+$ python3 dump.py dbname=ubuntu-it-chiedi
+```
+
+Il comando supporta tutti i parametri di connessione di psycopg2, quindi se ci
+si connette al database in modo diverso basta modificare quei parametri.

data/README.md

+# Archivio dei dati di Chiedi
+
+Questi sono i dati storici estratti da Chiedi, il portale di domande e risposte
+di Ubuntu-it, prima della sua chiusura. I dati sono disponibili in formato
+JSON, e sono rilasciati sotto licenza CreativeCommons BY-SA 3.0.
+
+## Domande poste sul portale
+
+Ogni domanda si trova in un file `.json` chiamato con l'ID della domanda, nella
+directory `questions/`. La struttura di un file è la seguente:
+
+```json
+{
+    "title": "Il titolo della domanda",
+    "body": "Testo della domanda in **Markdown",
+    "author": "username",
+    "score": 0,
+    "tags": ["tag1", "tag2"],
+    "created_at": "1970-01-01T00:00:00+00:00",
+    "comments": [
+        {
+            "author": "username",
+            "body": "Testo del commento in **Markdown**",
+            "created_at": "1970-01-01T00:00:00+00:00"
+        }
+    ],
+    "answers": [
+        {
+            "author": "username",
+            "body": "Testo della risposta in **Markdown**",
+            "score": 4,
+            "accepted": true,
+            "created_at": "1970-01-01T00:00:00+00:00",
+            "comments": [
+                {
+                    "author": "username",
+                    "body": "Testo del commento in **Markdown**",
+                    "created_at": "1970-01-01T00:00:00+00:00"
+                }
+            ]
+        }
+    ]
+}
+```

dump.py

+#!/usr/bin/env python3
+# Copyright (c) 2018 Pietro Albini <pietroalbini@ubuntu-it.org>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import json
+import os
+import sys
+
+import psycopg2
+import psycopg2.extras
+
+
+OUTPUT_DIRECTORY = "data"
+
+
+def progress(message, iterator):
+    total = len(iterator)
+    for i, item in enumerate(iterator):
+        if i % int(total / 100) == 0:
+            sys.stdout.write("\r%s %s%%" % (message, int(i * 100 / total)))
+        sys.stdout.flush()
+        yield item
+
+    sys.stdout.write("\r%s 100%%\n" % message)
+    sys.stdout.flush()
+
+
+class Dumper:
+
+    def __init__(self, connstring):
+        self.db = psycopg2.connect(connstring)
+
+    def query(self, sql, *params):
+        """Execute a query against the database"""
+        cursor = self.db.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
+        cursor.execute(sql, params)
+        return cursor.fetchall()
+
+    def dump_all(self, output_dir):
+        """Dump everything in the output directory"""
+        # Dump all the questions
+        os.makedirs(os.path.join(output_dir, "questions"), exist_ok=True)
+        ids = [row["id"] for row in self.query(
+            "SELECT id FROM forum_node WHERE node_type = 'question';"
+        )]
+
+        for id in progress("Dumping questions...", ids):
+            data = self.dump_question(id)
+            if data is None:
+                continue
+
+            path = os.path.join(output_dir, "questions", "%s.json" % id)
+            json.dump(
+                data, open(path, "w"), separators=(',',':'), sort_keys=True,
+            )
+
+    def dump_question(self, question_id):
+        """Dump a question as a JSON file"""
+        question = self.query(
+            "SELECT n.title, n.tagnames, r.body, n.score, n.added_at, "
+            "n.state_string, u.username, u.is_superuser "
+            "FROM forum_node n "
+            "INNER JOIN auth_user u ON (n.author_id = u.id) "
+            "INNER JOIN forum_noderevision r ON (n.active_revision_id = r.id) "
+            "WHERE n.id = %s;",
+            question_id
+        )[0]
+
+        if self.is_deleted(question):
+            return
+
+        return {
+            "title": question["title"],
+            "body": question["body"],
+            "tags": question["tagnames"].split(" "),
+            "score": question["score"],
+
+            "author": self.format_author(question),
+            "created_at": self.format_time(question["added_at"]),
+
+            "answers": self.dump_answers(question_id),
+            "comments": self.dump_comments(question_id),
+        }
+
+    def dump_answers(self, question_id):
+        """Dump all the answers of a question"""
+        answers = self.query(
+            "SELECT n.id, r.body, n.added_at, n.score, n.marked, "
+            "n.state_string, u.username, u.is_superuser "
+            "FROM forum_node n "
+            "INNER JOIN auth_user u ON (n.author_id = u.id) "
+            "INNER JOIN forum_noderevision r ON (n.active_revision_id = r.id) "
+            "WHERE n.node_type = 'answer' AND n.parent_id = %s "
+            "ORDER BY n.marked DESC, n.score DESC;",
+            question_id
+        )
+        result = []
+
+        for answer in answers:
+            if self.is_deleted(answer):
+                continue
+
+            result.append({
+                "body": answer["body"],
+                "score": answer["score"],
+                "accepted": answer["marked"],
+                "author": self.format_author(answer),
+                "created_at": self.format_time(answer["added_at"]),
+                "comments": self.dump_comments(answer["id"]),
+            })
+
+        return result
+
+    def dump_comments(self, parent_id):
+        """Dump all the comments on a node"""
+        comments = self.query(
+            "SELECT r.body, n.added_at, n.state_string, u.username, "
+            "u.is_superuser "
+            "FROM forum_node n "
+            "INNER JOIN auth_user u ON (n.author_id = u.id) "
+            "INNER JOIN forum_noderevision r ON (n.active_revision_id = r.id) "
+            "WHERE n.node_type = 'comment' AND n.parent_id = %s "
+            "ORDER BY n.added_at;",
+            parent_id
+        )
+        result = []
+
+        for comment in comments:
+            if self.is_deleted(comment):
+                continue
+
+            result.append({
+                "body": comment["body"],
+                "author": self.format_author(comment),
+                "created_at": self.format_time(comment["added_at"]),
+            })
+
+        return result
+
+    def is_deleted(self, node):
+        """Check if a node is deleted"""
+        return "(deleted)" in node["state_string"]
+
+    def format_author(self, data):
+        """Format the author username"""
+        if data["is_superuser"]:
+            return "%s (admin)" % data["username"]
+        else:
+            return data["username"]
+
+    def format_time(self, time):
+        """Format a datetime"""
+        return time.replace(microsecond=0).isoformat()
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: dump.py <psycopg2-connstring>")
+        exit(1)
+
+    dumper = Dumper(" ".join(sys.argv[1:]))
+    dumper.dump_all(OUTPUT_DIRECTORY)