Migrate secrets to environment variables

.foreman

@@ -0,0 +1,2 @@
+port: 8000
+procfile: Procfile-development

Procfile-development

@@ -0,0 +1 @@
+web: python manage.py runserver "0.0.0.0:$PORT" --settings "$DJANGO_SETTINGS_MODULE"

karmaworld/apps/notes/gdrive.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding:utf8 -*-
 # Copyright (C) 2012  FinalsClub Foundation
+import base64
 
 import datetime
 import logging
@@ -26,13 +27,14 @@ from apiclient.discovery import build
 from apiclient.http import MediaInMemoryUpload
 from oauth2client.client import SignedJwtAssertionCredentials
 
-import karmaworld.secret.drive as drive
-
 logger = logging.getLogger(__name__)
 
 PDF_MIMETYPE = 'application/pdf'
 PPT_MIMETYPES = ['application/vnd.ms-powerpoint', 'application/vnd.openxmlformats-officedocument.presentationml.presentation']
 
+GOOGLE_CLIENT_SECRETS = os.environ['GOOGLE_CLIENT_SECRETS']
+GOOGLE_SERVICE_KEY_BASE64 = os.environ['GOOGLE_SERVICE_KEY_BASE64']
+GOOGLE_USER = os.environ['GOOGLE_USER']
 
 def build_api_service():
     """
@@ -47,16 +49,13 @@ def build_api_service():
     """
 
     # Extract the service address from the client secret
-    with open(drive.CLIENT_SECRET, 'r') as fp:
-        service_user = json.load(fp)['web']['client_email']
+    service_user = json.loads(GOOGLE_CLIENT_SECRETS)['web']['client_email']
 
     # Pull in the service's p12 private key.
-    with open(drive.SERVICE_KEY, 'rb') as p12:
-        # Use the private key to auth as the service user for access to the
-        # Google Drive of the GOOGLE_USER
-        credentials = SignedJwtAssertionCredentials(service_user, p12.read(),
+    p12 = base64.decodestring(GOOGLE_SERVICE_KEY_BASE64)
+    credentials = SignedJwtAssertionCredentials(service_user, p12,
                                scope='https://www.googleapis.com/auth/drive',
-                               sub=drive.GOOGLE_USER)
+                               sub=GOOGLE_USER)
 
     return build('drive', 'v2', http=credentials.authorize(httplib2.Http()))
 

karmaworld/apps/notes/management/commands/import_ocw_json.py

@@ -6,18 +6,17 @@ import json
 import os.path
 import requests
 
-from apps.notes.models import Note
-from apps.notes.gdrive import convert_raw_document
-from apps.courses.models import Course
-from apps.courses.models import School
-from apps.courses.models import Professor
-from apps.courses.models import Department
-from apps.courses.models import ProfessorTaught
-from apps.courses.models import ProfessorAffiliation
-from apps.licenses.models import License
-from apps.document_upload.models import RawDocument
+from karmaworld.apps.notes.models import Note
+from karmaworld.apps.notes.gdrive import convert_raw_document
+from karmaworld.apps.courses.models import Course
+from karmaworld.apps.courses.models import School
+from karmaworld.apps.courses.models import Professor
+from karmaworld.apps.courses.models import Department
+from karmaworld.apps.licenses.models import License
+from karmaworld.apps.document_upload.models import RawDocument
 from django.core.management.base import BaseCommand
-from karmaworld.secret.filepicker import FILEPICKER_API_KEY
+
+FILEPICKER_API_KEY = os.environ['FILEPICKER_API_KEY']
 
 class Command(BaseCommand):
     args = 'directory containing json files from mit-ocw-scraper'

karmaworld/apps/notes/management/commands/populate_filepicker.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding:utf8 -*-
 # Copyright (C) 2014  FinalsClub Foundation
+import os
 import time
 import json
 
@@ -10,7 +11,8 @@ from django.utils.text import slugify
 from karmaworld.apps.notes.models import Note
 from karmaworld.utils.filepicker import sign_fp_policy, encode_fp_policy
 import requests
-from karmaworld.secret.filepicker import FILEPICKER_API_KEY
+
+FILEPICKER_API_KEY = os.environ['FILEPICKER_API_KEY']
 
 
 class Command(BaseCommand):

karmaworld/apps/notes/models.py

@@ -20,7 +20,6 @@ from django.db.models import SET_NULL
 from django.db.models.signals import post_save, post_delete, pre_save
 from django.dispatch import receiver
 from karmaworld.apps.users.models import NoteKarmaEvent, GenericKarmaEvent
-from karmaworld.secret.filepicker import FILEPICKER_API_KEY
 from karmaworld.utils.filepicker import encode_fp_policy, sign_fp_policy
 import os
 import time
@@ -40,6 +39,8 @@ from karmaworld.apps.licenses.models import License
 from karmaworld.apps.notes.search import SearchIndex
 from karmaworld.settings.manual_unique_together import auto_add_check_unique_together
 
+FILEPICKER_API_KEY = os.environ['FILEPICKER_API_KEY']
+
 ANONYMOUS_UPLOAD_URLS = 'anonymous_upload_urls'
 KEYWORD_MTURK_THRESHOLD = 3
 

karmaworld/apps/notes/search.py

@@ -3,13 +3,13 @@
 # Copyright (C) 2013  FinalsClub Foundation
 
 import calendar
+import os
 import time
 import uuid
 from django.core.exceptions import ImproperlyConfigured
 
 import indextank.client as itc
 from django.conf import settings
-import karmaworld.secret.indexden as secret
 
 import logging
 
@@ -20,6 +20,8 @@ MOCK_MODE = settings.TESTING
 logging.basicConfig()
 logger = logging.getLogger(__name__)
 
+INDEXDEN_INDEX = os.environ['INDEXDEN_INDEX']
+INDEXDEN_PRIVATE_URL = os.environ['INDEXDEN_PRIVATE_URL']
 
 class SearchResult(object):
     """The result of making a query into IndexDen.
@@ -57,7 +59,7 @@ class SearchIndex(object):
     __metaclass__ = Singleton
 
     def __init__(self):
-        self.index_name = secret.INDEX
+        self.index_name = INDEXDEN_INDEX
 
         # If we're in production mode,
         # or if we're in testing mode with indexing
@@ -66,7 +68,7 @@ class SearchIndex(object):
         if MOCK_MODE:
             return
 
-        self.api_client = itc.ApiClient(secret.PRIVATE_URL)
+        self.api_client = itc.ApiClient(INDEXDEN_PRIVATE_URL)
         if not self.api_client.get_index(self.index_name).exists():
             time.sleep(5)
             self.api_client.create_index(self.index_name, {'public_search': False})

karmaworld/apps/notes/tasks.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding:utf8 -*-
 # Copyright (C) 2013  FinalsClub Foundation
+import os
 
 import traceback
 from celery import task
@@ -16,16 +17,19 @@ def tweet_note():
     """Tweet about a new note."""
 
     try:
-        import karmaworld.secret.twitter as secrets
-    except ImportError:
+        TWITTER_CONSUMER_KEY = os.environ['TWITTER_CONSUMER_KEY']
+        TWITTER_CONSUMER_SECRET = os.environ['TWITTER_CONSUMER_SECRET']
+        TWITTER_ACCESS_TOKEN_KEY = os.environ['TWITTER_ACCESS_TOKEN_KEY']
+        TWITTER_ACCESS_TOKEN_SECRET = os.environ['TWITTER_ACCESS_TOKEN_SECRET']
+    except:
         logger.warn("No twitter secrets found, not running tweet_note")
         return
 
     try:
-        api = twitter.Api(consumer_key=secrets.CONSUMER_KEY,
-                          consumer_secret=secrets.CONSUMER_SECRET,
-                          access_token_key=secrets.ACCESS_TOKEN_KEY,
-                          access_token_secret=secrets.ACCESS_TOKEN_SECRET)
+        api = twitter.Api(consumer_key=TWITTER_CONSUMER_KEY,
+                          consumer_secret=TWITTER_CONSUMER_SECRET,
+                          access_token_key=TWITTER_ACCESS_TOKEN_KEY,
+                          access_token_secret=TWITTER_ACCESS_TOKEN_SECRET)
 
         newest_notes = Note.objects.all().order_by('-uploaded_at')[:100]
         for n in newest_notes:

karmaworld/apps/quizzes/tasks.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding:utf8 -*-
 # Copyright (C) 2013  FinalsClub Foundation
+import os
 from boto.mturk.qualification import PercentAssignmentsApprovedRequirement, Qualifications
 from boto.mturk.question import Overview, FormattedContent, QuestionContent, Question, FreeTextAnswer, QuestionForm, \
     AnswerSpecification
@@ -11,6 +12,7 @@ from django.contrib.sites.models import Site
 from django.core.exceptions import ObjectDoesNotExist
 from karmaworld.apps.notes.models import Note
 from karmaworld.apps.quizzes.models import Keyword
+from django.conf import settings
 
 logger = get_task_logger(__name__)
 
@@ -67,12 +69,12 @@ def submit_extract_keywords_hit(note):
     choose keywords and definitions from the given note."""
 
     try:
-        from karmaworld.secret.mturk import AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, MTURK_HOST
-    except ImportError:
+        MTURK_HOST = os.environ['MTURK_HOST']
+    except:
         logger.warn('Could not find Mechanical Turk secrets, not running submit_extract_keywords_hit')
         return
 
-    connection = MTurkConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
+    connection = MTurkConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY,
                                  host=MTURK_HOST)
 
     overview = Overview()

karmaworld/secret/client_secrets.json.example

@@ -1,10 +0,0 @@
-{
-  "web": {
-    "auth_uri": "https://accounts.google.com/o/oauth2/auth",
-    "token_uri": "https://accounts.google.com/o/oauth2/token",
-    "client_email": "PROJECT@developer.gserviceaccount.com",
-    "client_x509_cert_url":"https://www.googleapis.com/robot/v1/metadata/x509/PROJECT@developer.gserviceaccount.com",
-    "client_id":"PROJECT.apps.googleusercontent.com",
-    "auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs"
-  }
-}

karmaworld/secret/db_settings.py.example

@@ -1,11 +0,0 @@
-#!/usr/bin/env python
-# -*- coding:utf8 -*-
-# Copyright (C) 2012  FinalsClub Foundation
-"""
-DO NOT modify this file directly, instead copy it to db_settings.py and modify
-that file.
-DO NOT check db_settings.py into source control.
-"""
-PROD_DB_NAME = 'DB_NAME'
-PROD_DB_USERNAME = 'DB_USER_NAME'
-PROD_DB_PASSWORD = 'PASSWD'

karmaworld/secret/drive.py.example

@@ -1,20 +0,0 @@
-#!/usr/bin/env python
-# -*- coding:utf8 -*-
-# Copyright (C) 2012  FinalsClub Foundation
-"""
-DO NOT modify this file directly, instead copy it to drive.py and modify
-that file.
-DO NOT check drive.py into source control.
-"""
-
-import os
-
-from django.conf import settings
-
-
-CLIENT_SECRET = os.path.join(settings.DJANGO_ROOT, \
-                             'path/to/client_secrets.json')
-
-SERVICE_KEY = os.path.join(settings.DJANGO_ROOT, 'path/to/some.p12')
-
-GOOGLE_USER = 'GOOGLE_APPS_EMAIL_ADDRESS'

karmaworld/secret/email.py.example

@@ -1,5 +0,0 @@
-
-STMP_HOST = ''
-SMTP_USERNAME = ''
-SMTP_PASSWORD = ''
-

karmaworld/secret/filepicker.py.example

@@ -1,10 +0,0 @@
-#!/usr/bin/env python
-# -*- coding:utf8 -*-
-# Copyright (C) 2012  FinalsClub Foundation
-"""
-DO NOT modify this file directly, instead copy it to filepicker.py and modify
-that file.
-DO NOT check filepicker.py into source control.
-"""
-FILEPICKER_API_KEY = 0
-SECRET = 0

karmaworld/secret/indexden.py.example

@@ -1,3 +0,0 @@
-
-PRIVATE_URL = ''
-INDEX = ''

karmaworld/secret/mturk.py.example

@@ -1,7 +0,0 @@
-#!/usr/bin/env python
-# -*- coding:utf8 -*-
-# Copyright (C) 2014  FinalsClub Foundation
-
-AWS_ACCESS_KEY_ID = ''
-AWS_SECRET_ACCESS_KEY = ''
-MTURK_HOST = 'mechanicalturk.sandbox.amazonaws.com'

karmaworld/secret/static_s3.py.example

@@ -1,10 +0,0 @@
-#!/usr/bin/env python
-# -*- coding:utf8 -*-
-# Copyright (C) 2012  FinalsClub Foundation
-""" Dummy S3 bucket conf. file (for pushing static files """
-
-DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage'
-AWS_ACCESS_KEY_ID = 'access_id'
-AWS_SECRET_ACCESS_KEY = 'access_key'
-AWS_STORAGE_BUCKET_NAME = 'bucket'
-S3_URL = '//%s.s3.amazonaws.com/' % AWS_STORAGE_BUCKET_NAME

karmaworld/secret/twitter.py.example

@@ -1,8 +0,0 @@
-
-
-CONSUMER_KEY = ''
-CONSUMER_SECRET = ''
-ACCESS_TOKEN_KEY = ''
-ACCESS_TOKEN_SECRET = ''
-
-

karmaworld/settings/common.py

@@ -3,18 +3,16 @@
 # Copyright (C) 2012  FinalsClub Foundation
 """ Common settings and globals. """
 from datetime import timedelta
+from os.path import dirname, abspath, basename, normpath, join
 import sys
-from os.path import abspath, basename, dirname, join, normpath
+import os
 from sys import path
 from djcelery import setup_loader
 import dj_database_url
-from karmaworld.secret.filepicker import FILEPICKER_API_KEY as fp_api
 
-FILEPICKER_API_KEY = fp_api
+FILEPICKER_API_KEY = os.environ['FILEPICKER_API_KEY']
 FILEPICKER_INPUT_TYPE = 'filepicker'
 
-from karmaworld.secret.static_s3 import *
-
 SERIALIZATION_MODULES = {'json-pretty': 'karmaworld.apps.serializers.json_pretty'}
 
 
@@ -22,12 +20,7 @@ SERIALIZATION_MODULES = {'json-pretty': 'karmaworld.apps.serializers.json_pretty
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-ALLOWED_HOSTS
 # The hosts that this server runs from.
 ALLOWED_HOSTS = [
-    '127.0.0.1', # for dev systems / VMs, but should be safe enough
-    'localhost', # for dev systems / VMs, but should be safe enough
-    'beta.karmanotes.org',
-    'www.karmanotes.org',
-    'karmanotes.org',
-    'quiz.karmanotes.org',
+    '*'
 ]
 ########## END SECURITY CONFIGURATION
 
@@ -115,6 +108,14 @@ STATICFILES_FINDERS = (
     'django.contrib.staticfiles.finders.AppDirectoriesFinder',
     'compressor.finders.CompressorFinder',
 )
+
+S3_URL = '//%s.s3.amazonaws.com/' % os.environ.get('AWS_STORAGE_BUCKET_NAME')
+DEFAULT_FILE_STORAGE = os.environ['DEFAULT_FILE_STORAGE']
+
+AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID')
+AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
+AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME')
+
 ########## END STATIC FILE CONFIGURATION
 
 

karmaworld/settings/prod.py

@@ -10,9 +10,9 @@ from common import *
 
 try:
     # Include email is settings are there
-    from karmaworld.secret.email import SMTP_HOST
-    from karmaworld.secret.email import SMTP_USERNAME
-    from karmaworld.secret.email import SMTP_PASSWORD
+    SMTP_HOST = os.environ['SMTP_HOST']
+    SMTP_USERNAME = os.environ['SMTP_USERNAME']
+    SMTP_PASSWORD = os.environ['SMTP_PASSWORD']
     EMAIL = True
 except:
     EMAIL = False
@@ -128,7 +128,7 @@ AWS_HEADERS = {
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#static-url
 # S3_URL comes from karmaworld.secret.static_s3
-STATIC_URL = CLOUDFRONT_URL + AWS_LOCATION + '/'
+STATIC_URL = os.environ.get('CLOUDFRONT_URL') + AWS_LOCATION + '/'
 ########## END STORAGE CONFIGURATION
 
 ########## SSL FORWARDING CONFIGURATION
@@ -140,10 +140,10 @@ SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
 COMPRESS_OFFLINE = True
 
 # See: http://django_compressor.readthedocs.org/en/latest/settings/#django.conf.settings.COMPRESS_STORAGE
-COMPRESS_STORAGE = DEFAULT_FILE_STORAGE
+COMPRESS_STORAGE = os.environ.get('DEFAULT_FILE_STORAGE')
 
 # Make sure that django-compressor serves from CloudFront
-AWS_S3_CUSTOM_DOMAIN = CLOUDFRONT_DOMAIN
+AWS_S3_CUSTOM_DOMAIN = os.environ.get('CLOUDFRONT_DOMAIN')
 
 # See: http://django_compressor.readthedocs.org/en/latest/settings/#django.conf.settings.COMPRESS_CSS_FILTERS
 COMPRESS_CSS_FILTERS += [

karmaworld/utils/filepicker.py

@@ -3,7 +3,9 @@ import json
 import base64
 
 from hashlib import sha256
-from karmaworld.secret.filepicker import SECRET
+import os
+
+FILEPCIKER_SECRET = os.environ['FILEPCIKER_SECRET']
 
 def encode_fp_policy(policy):
     """ Return URL-safe Base64 encoded JSON Filepicker policy. """
@@ -23,6 +25,6 @@ def sign_fp_policy(policy):
     """ Return a signature appropriate for the given encoded policy. """
     # https://developers.inkfilepicker.com/docs/security/#signPolicy
     # hash it up, bra!
-    engine = hmac.new(SECRET, digestmod=sha256)
+    engine = hmac.new(FILEPCIKER_SECRET, digestmod=sha256)
     engine.update(policy)
     return engine.hexdigest()

manage.py

@@ -7,7 +7,7 @@ if __name__ == "__main__":
     PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__))
     sys.path.append(os.path.join(PROJECT_ROOT, 'karmaworld/apps'))
 
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "karmaworld.settings.prod")
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "karmaworld.settings.dev")
 
     from django.core.management import execute_from_command_line
 

reqs/common.txt

@@ -1,31 +0,0 @@
-Django>=1.5.0,<1.6.0
-django-celery==3.0.21
-django-compressor==1.2
-Fabric==1.5.3
-South==0.8.4
-supervisor==3.0b1
-oauth2client==1.0
-urllib3==1.5
-google-api-python-client==1.0
-django-grappelli==2.4.8
-django-taggit
-git+https://github.com/btbonval/django-filepicker.git
-filemagic==1.6
-requests
-beautifulsoup4
-pyopenssl
-python-twitter
-gdshortener
-git+https://github.com/flaptor/indextank-py.git
-git+https://github.com/jhilker/html2text.git
-django-allauth
-boto
-django-storages==1.1.4
-django-reversion
-django-ajax-selects
-git+https://github.com/btbonval/django-ajax-selects-cascade.git
-psycopg2
-pyth
-querystring_parser
-gunicorn
-dj-database-url

reqs/prod.txt

@@ -1,5 +0,0 @@
--r common.txt
-cssmin==0.1.4
-pylibmc==1.2.3
-jsmin==2.0.2
-amqplib==1.0.2

reqs/vmdev.txt

@@ -1,11 +0,0 @@
--r prod.txt
-django-debug-toolbar
-django-debug-toolbar-template-timings
-ipython==0.13.1
-ipdb==0.7
-django-extensions==1.0.3 # some extra debugging tools for manage.py
-# Testing requirements
-django-nose==1.1
-coverage==3.6
-selenium==2.38.4
-mock

reqs/dev.txt → requirements-dev.txt

@@ -1,4 +1,3 @@
--r common.txt
 django-kombu==0.9.4
 django-debug-toolbar
 django-debug-toolbar-template-timings

requirements.txt

@@ -29,14 +29,6 @@ pyth
 querystring_parser
 gunicorn
 dj-database-url
-django-kombu==0.9.4
-django-debug-toolbar
-ipdb==0.7
-django-extensions==1.0.3 # some extra debugging tools for manage.py
-# Testing requirements
-django-nose==1.1
-coverage==3.6
-selenium==2.38.4
 mock
 cssmin==0.1.4
 amqplib==1.0.2