From 22c5da1629cb18c4eb20c1ff49d4e727dc0c4a8d Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 12:40:44 -0400 Subject: [PATCH 01/29] CC-3755: Found Exception in apache's error log -Return error code so that python services automatically retry later. --- airtime_mvc/application/models/Preference.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/airtime_mvc/application/models/Preference.php b/airtime_mvc/application/models/Preference.php index c9c1f17a3..90e87b9b2 100644 --- a/airtime_mvc/application/models/Preference.php +++ b/airtime_mvc/application/models/Preference.php @@ -61,7 +61,13 @@ class Application_Model_Preference ." VALUES ($id, '$key', $value)"; } } - return $con->exec($sql); + try { + $con->exec($sql); + } catch (Exception $e){ + Logging::log("Could not connect to database."); + header('HTTP/1.0 503 Service Unavailable'); + exit; + } } public static function GetValue($key, $isUserValue = false){ From 9092b7f7258de241b91f781257998217ae1f5d75 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 12:47:29 -0400 Subject: [PATCH 02/29] CC-3764: Media Library->Edit Metadata: Throw exception when trying to edit metadata -fixed --- airtime_mvc/application/models/StoredFile.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airtime_mvc/application/models/StoredFile.php b/airtime_mvc/application/models/StoredFile.php index 886f9a8ba..a6044c868 100644 --- a/airtime_mvc/application/models/StoredFile.php +++ b/airtime_mvc/application/models/StoredFile.php @@ -235,7 +235,9 @@ class Application_Model_StoredFile { foreach ($c['user'] as $constant => $value) { if (preg_match('/^MDATA_KEY/', $constant)) { if (isset($dbmd_copy[$value])) { - $md[$constant] = $this->getDbColMetadataValue($value); + $propelColumn = $dbmd_copy[$value]; + $method = "get$propelColumn"; + $md[$constant] = $this->_file->$method(); } } } From ebc8af0863e727ed73a1aa23fa4e1b603044ed28 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 13:03:05 -0400 Subject: [PATCH 03/29] CC-3749: Media Monitor should not load files pypo cannot read --- .../media-monitor/airtimefilemonitor/mediamonitorcommon.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py index b28b1dfcd..0c38a7d45 100644 --- a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py +++ b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py @@ -84,6 +84,7 @@ class MediaMonitorCommon: # stat.S_IROTH - readable by all permission # stat.S_IXOTH - excutable by all permission # try to set permission + self.logger.info("%s has incorrect permissions. Will modify to be readable.", item) if self.is_parent_directory(item, self.config.storage_directory) or self.is_parent_directory(item, self.config.imported_directory) or self.is_parent_directory(item, self.config.organize_directory): if is_dir is True: os.chmod(item, 02777) @@ -98,8 +99,7 @@ class MediaMonitorCommon: bitor = stats.st_mode | stat.S_IROTH os.chmod(item, bitor) except Exception, e: - self.logger.error("Failed to change file's owner/group/permissions. %s", e) - self.logger.error("traceback: %s", traceback.format_exc()) + self.logger.warn("Failed to change owner/group/permissions for %s", item) return False finally: os.umask(omask) From 08217d59fc0986ba76637523a210aa099afde1cb Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 13:23:46 -0400 Subject: [PATCH 04/29] CC-3749: Media Monitor should not load files pypo cannot read -fixed --- .../airtimefilemonitor/mediamonitorcommon.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py index 0c38a7d45..89800c5c5 100644 --- a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py +++ b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py @@ -79,18 +79,24 @@ class MediaMonitorCommon: def set_needed_file_permissions(self, item, is_dir): try: omask = os.umask(0) - if not self.has_correct_permissions(item, 'www-data', 'www-data') or not self.has_correct_permissions(item, 'pypo', 'pypo'): + if not self.has_correct_permissions(item, 'www-data', 'www-data') \ + or not self.has_correct_permissions(item, 'pypo', 'pypo'): + # stats.st_mode is the original permission # stat.S_IROTH - readable by all permission # stat.S_IXOTH - excutable by all permission # try to set permission - self.logger.info("%s has incorrect permissions. Will modify to be readable.", item) - if self.is_parent_directory(item, self.config.storage_directory) or self.is_parent_directory(item, self.config.imported_directory) or self.is_parent_directory(item, self.config.organize_directory): + self.logger.warn("%s has incorrect permissions for reading. Skipping import.", item) + """ + if self.is_parent_directory(item, self.config.storage_directory) \ + or self.is_parent_directory(item, self.config.imported_directory) \ + or self.is_parent_directory(item, self.config.organize_directory): + if is_dir is True: os.chmod(item, 02777) else: os.chmod(item, 0666) - else : + else: # add world readable permission stats = os.stat(item) if is_dir is True: @@ -98,6 +104,7 @@ class MediaMonitorCommon: else: bitor = stats.st_mode | stat.S_IROTH os.chmod(item, bitor) + """ except Exception, e: self.logger.warn("Failed to change owner/group/permissions for %s", item) return False From fa373a683ea95da76b05a6244cffd4fc270518ff Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 13:37:43 -0400 Subject: [PATCH 05/29] CC-3749: Media Monitor should not load files pypo cannot read -fixed --- .../airtimefilemonitor/airtimenotifier.py | 12 +++---- .../airtimefilemonitor/airtimeprocessevent.py | 6 ++-- .../airtimefilemonitor/mediamonitorcommon.py | 33 +++---------------- 3 files changed, 14 insertions(+), 37 deletions(-) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py b/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py index 42f22b0ba..cbb404712 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py @@ -103,9 +103,9 @@ class AirtimeNotifier(Notifier): self.mmc.ensure_is_dir(self.config.imported_directory) self.mmc.ensure_is_dir(self.config.organize_directory) - self.mmc.set_needed_file_permissions(self.config.storage_directory, True) - self.mmc.set_needed_file_permissions(self.config.imported_directory, True) - self.mmc.set_needed_file_permissions(self.config.organize_directory, True) + self.mmc.is_readable(self.config.storage_directory, True) + self.mmc.is_readable(self.config.imported_directory, True) + self.mmc.is_readable(self.config.organize_directory, True) self.watch_directory(new_storage_directory) elif m['event_type'] == "file_delete": @@ -192,17 +192,17 @@ class AirtimeNotifier(Notifier): mm = self.proc_fun() - self.mmc.set_needed_file_permissions(directory, True) + self.mmc.is_readable(directory, True) for (path, dirs, files) in os.walk(directory): for d in dirs: - self.mmc.set_needed_file_permissions(os.path.join(path, d), True) + self.mmc.is_readable(os.path.join(path, d), True) for filename in files: full_filepath = os.path.join(path, filename) if self.mmc.is_audio_file(full_filepath): - if self.mmc.set_needed_file_permissions(full_filepath, False): + if self.mmc.is_readable(full_filepath, False): self.logger.info("importing %s", full_filepath) event = {'filepath': full_filepath, 'mode': self.config.MODE_CREATE, 'is_recorded_show': False} mm.multi_queue.put(event) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py index 0d6a6e602..947b16282 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py @@ -151,14 +151,14 @@ class AirtimeProcessEvent(ProcessEvent): self.logger.error('Exception: %s', e) self.logger.error("traceback: %s", traceback.format_exc()) - self.mmc.set_needed_file_permissions(pathname, dir) + self.mmc.is_readable(pathname, dir) is_recorded = self.mmc.is_parent_directory(pathname, self.config.recorded_directory) self.file_events.append({'mode': self.config.MODE_CREATE, 'filepath': pathname, 'is_recorded_show': is_recorded}) else: #event is because of a created directory if self.mmc.is_parent_directory(pathname, self.config.storage_directory): - self.mmc.set_needed_file_permissions(pathname, dir) + self.mmc.is_readable(pathname, dir) def process_IN_MODIFY(self, event): # if IN_MODIFY is followed by IN_CREATE, it's not true modify event @@ -237,7 +237,7 @@ class AirtimeProcessEvent(ProcessEvent): if event.pathname in filename: self.handle_mount_change() #if stuff dropped in stor via a UI move must change file permissions. - self.mmc.set_needed_file_permissions(event.pathname, event.dir) + self.mmc.is_readable(event.pathname, event.dir) if not event.dir: if self.mmc.is_audio_file(event.name): if event.cookie in self.temp_files: diff --git a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py index 89800c5c5..b6ed45a80 100644 --- a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py +++ b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py @@ -49,7 +49,7 @@ class MediaMonitorCommon: return False #check if file is readable by "nobody" - def has_correct_permissions(self, filepath, euid='nobody', egid='nogroup'): + def is_user_readable(self, filepath, euid='nobody', egid='nogroup'): try: uid = pwd.getpwnam(euid)[2] @@ -76,40 +76,17 @@ class MediaMonitorCommon: return readable # the function only changes the permission if its not readable by www-data - def set_needed_file_permissions(self, item, is_dir): + def is_readable(self, item, is_dir): try: - omask = os.umask(0) - if not self.has_correct_permissions(item, 'www-data', 'www-data') \ - or not self.has_correct_permissions(item, 'pypo', 'pypo'): + if not self.is_user_readable(item, 'www-data', 'www-data') \ + or not self.is_user_readable(item, 'pypo', 'pypo'): - # stats.st_mode is the original permission - # stat.S_IROTH - readable by all permission - # stat.S_IXOTH - excutable by all permission - # try to set permission self.logger.warn("%s has incorrect permissions for reading. Skipping import.", item) - """ - if self.is_parent_directory(item, self.config.storage_directory) \ - or self.is_parent_directory(item, self.config.imported_directory) \ - or self.is_parent_directory(item, self.config.organize_directory): - - if is_dir is True: - os.chmod(item, 02777) - else: - os.chmod(item, 0666) - else: - # add world readable permission - stats = os.stat(item) - if is_dir is True: - bitor = stats.st_mode | stat.S_IROTH | stat.S_IXOTH - else: - bitor = stats.st_mode | stat.S_IROTH - os.chmod(item, bitor) - """ + return False except Exception, e: self.logger.warn("Failed to change owner/group/permissions for %s", item) return False finally: - os.umask(omask) return True From d6a12053eb02d13dcbbdc8e2a0cd3c9dc82b12ec Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 13:39:48 -0400 Subject: [PATCH 06/29] CC-3752: Media Monitor DB sync -forgot to exit --- python_apps/media-monitor/media_monitor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python_apps/media-monitor/media_monitor.py b/python_apps/media-monitor/media_monitor.py index d9dcf518c..7c39bba0a 100644 --- a/python_apps/media-monitor/media_monitor.py +++ b/python_apps/media-monitor/media_monitor.py @@ -51,6 +51,7 @@ def configure_locale(): if current_locale_encoding not in ['utf-8', 'utf8']: logger.error("Need a UTF-8 locale. Currently '%s'. Exiting..." % current_locale_encoding) + sys.exit(1) # configure logging try: From e13d06df4553d0d698d8abf25ad918dd86b3c02b Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 13:45:43 -0400 Subject: [PATCH 07/29] CC-3749: Media Monitor should not load files pypo cannot read -fixed --- .../airtimefilemonitor/airtimenotifier.py | 4 ---- .../airtimefilemonitor/mediamonitorcommon.py | 12 +++--------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py b/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py index cbb404712..c6dac73c5 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py @@ -194,10 +194,6 @@ class AirtimeNotifier(Notifier): self.mmc.is_readable(directory, True) for (path, dirs, files) in os.walk(directory): - - for d in dirs: - self.mmc.is_readable(os.path.join(path, d), True) - for filename in files: full_filepath = os.path.join(path, filename) diff --git a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py index b6ed45a80..8e68b88bf 100644 --- a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py +++ b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py @@ -78,17 +78,11 @@ class MediaMonitorCommon: # the function only changes the permission if its not readable by www-data def is_readable(self, item, is_dir): try: - if not self.is_user_readable(item, 'www-data', 'www-data') \ - or not self.is_user_readable(item, 'pypo', 'pypo'): - - self.logger.warn("%s has incorrect permissions for reading. Skipping import.", item) - return False + return self.is_user_readable(item, 'www-data', 'www-data') \ + and not self.is_user_readable(item, 'pypo', 'pypo'): except Exception, e: - self.logger.warn("Failed to change owner/group/permissions for %s", item) + self.logger.warn("Failed to check owner/group/permissions for %s", item) return False - finally: - return True - #checks if path is a directory, and if it doesnt exist, then creates it. #Otherwise prints error to log file. From 7e9d1a3308ecfa9bf6c933aec51fc61488601476 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 13:50:38 -0400 Subject: [PATCH 08/29] CC-3749: Media Monitor should not load files pypo cannot read -fixed --- .../media-monitor/airtimefilemonitor/mediamonitorcommon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py index 8e68b88bf..7d9069c05 100644 --- a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py +++ b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py @@ -79,7 +79,7 @@ class MediaMonitorCommon: def is_readable(self, item, is_dir): try: return self.is_user_readable(item, 'www-data', 'www-data') \ - and not self.is_user_readable(item, 'pypo', 'pypo'): + and not self.is_user_readable(item, 'pypo', 'pypo') except Exception, e: self.logger.warn("Failed to check owner/group/permissions for %s", item) return False From d98d23d483aafeed85c1292a0fd31ea9a0f050ea Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 15:04:54 -0400 Subject: [PATCH 09/29] CC-3749: Media Monitor should not load files pypo cannot read -fixed --- .../media-monitor/airtimefilemonitor/airtimenotifier.py | 1 - .../media-monitor/airtimefilemonitor/airtimeprocessevent.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py b/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py index c6dac73c5..8986ffc6f 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py @@ -150,7 +150,6 @@ class AirtimeNotifier(Notifier): file_md = None data = None - if (os.path.exists(filepath) and (mode == self.config.MODE_CREATE)): if file_md is None: mutagen = self.md_manager.get_md_from_file(filepath) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py index 947b16282..019f460e0 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py @@ -134,7 +134,7 @@ class AirtimeProcessEvent(ProcessEvent): #file is being overwritten/replaced in GUI. elif "goutputstream" in pathname: self.temp_files[pathname] = None - elif self.mmc.is_audio_file(pathname): + elif self.mmc.is_audio_file(pathname) and self.mmc.is_readable(pathname, False): if self.mmc.is_parent_directory(pathname, self.config.organize_directory): #file was created in /srv/airtime/stor/organize. Need to process and move #to /srv/airtime/stor/imported @@ -173,7 +173,7 @@ class AirtimeProcessEvent(ProcessEvent): # update timestamp on create_dict for the entry with pathname as the key if pathname in self.create_dict: self.create_dict[pathname] = time.time() - if not dir and not self.mmc.is_parent_directory(pathname, self.config.organize_directory): + if not dir and not self.mmc.is_parent_directory(pathname, self.config.organize_directory) and self.mmc.is_readable(pathname, False): self.logger.info("Modified: %s", pathname) if self.mmc.is_audio_file(name): self.file_events.append({'filepath': pathname, 'mode': self.config.MODE_MODIFY}) @@ -239,7 +239,7 @@ class AirtimeProcessEvent(ProcessEvent): #if stuff dropped in stor via a UI move must change file permissions. self.mmc.is_readable(event.pathname, event.dir) if not event.dir: - if self.mmc.is_audio_file(event.name): + if self.mmc.is_audio_file(event.name) and self.mmc.is_readable(full_filepath, False): if event.cookie in self.temp_files: self.file_events.append({'filepath': event.pathname, 'mode': self.config.MODE_MODIFY}) del self.temp_files[event.cookie] From 906b6e8bd0c7d118d962766741e01603e1d0ca6e Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 15:22:54 -0400 Subject: [PATCH 10/29] CC-3749: Media Monitor should not load files pypo cannot read -logic error --- .../media-monitor/airtimefilemonitor/mediamonitorcommon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py index 7d9069c05..fdd83de6b 100644 --- a/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py +++ b/python_apps/media-monitor/airtimefilemonitor/mediamonitorcommon.py @@ -79,7 +79,7 @@ class MediaMonitorCommon: def is_readable(self, item, is_dir): try: return self.is_user_readable(item, 'www-data', 'www-data') \ - and not self.is_user_readable(item, 'pypo', 'pypo') + and self.is_user_readable(item, 'pypo', 'pypo') except Exception, e: self.logger.warn("Failed to check owner/group/permissions for %s", item) return False From 676205bb00ee08694cee9720e4cf827e68d22d3b Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 15:49:33 -0400 Subject: [PATCH 11/29] CC-3750Media Library->Edit Metadata: media-monitor throw exception when trying to edit metadata with non-ascii characters -Improved save metadata function --- .../airtimefilemonitor/airtimemetadata.py | 17 ++++++++--------- .../airtimefilemonitor/airtimenotifier.py | 2 +- .../airtimefilemonitor/airtimeprocessevent.py | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py b/python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py index cab57381c..1cef988b1 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimemetadata.py @@ -88,17 +88,16 @@ class AirtimeMetadata: try: airtime_file = mutagen.File(m['MDATA_KEY_FILEPATH'], easy=True) - for key in m.keys() : + for key in m: if key in self.airtime2mutagen: value = m[key] - if (value is not None): - self.logger.debug("Saving %s to file", key) - self.logger.debug(value) - if isinstance(value, basestring) and (len(value) > 0): - airtime_file[self.airtime2mutagen[key]] = api_client.encode_to(value, 'utf-8') - elif isinstance(value, int): - airtime_file[self.airtime2mutagen[key]] = str(value) - + + if value is not None: + value = unicode(value) + + if len(value) > 0: + self.logger.debug("Saving key '%s' with value '%s' to file", key, value) + airtime_file[self.airtime2mutagen[key]] = value airtime_file.save() except Exception, e: diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py b/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py index 8986ffc6f..e527d56cb 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimenotifier.py @@ -62,7 +62,7 @@ class AirtimeNotifier(Notifier): message.ack() self.logger.info("Received md from RabbitMQ: " + body) - m = json.loads(message.body) + m = json.loads(message.body) if m['event_type'] == "md_update": self.logger.info("AIRTIME NOTIFIER md update event") diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py index 019f460e0..419bbd685 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py @@ -173,7 +173,7 @@ class AirtimeProcessEvent(ProcessEvent): # update timestamp on create_dict for the entry with pathname as the key if pathname in self.create_dict: self.create_dict[pathname] = time.time() - if not dir and not self.mmc.is_parent_directory(pathname, self.config.organize_directory) and self.mmc.is_readable(pathname, False): + if not dir and not self.mmc.is_parent_directory(pathname, self.config.organize_directory): self.logger.info("Modified: %s", pathname) if self.mmc.is_audio_file(name): self.file_events.append({'filepath': pathname, 'mode': self.config.MODE_MODIFY}) From 4d7d17fe553f7e4d5d9691d7cad802cc006f8f3e Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 17:25:59 -0400 Subject: [PATCH 12/29] fix invalid file variable name --- .../media-monitor/airtimefilemonitor/airtimeprocessevent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py index 419bbd685..14c5ec64d 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py @@ -239,7 +239,7 @@ class AirtimeProcessEvent(ProcessEvent): #if stuff dropped in stor via a UI move must change file permissions. self.mmc.is_readable(event.pathname, event.dir) if not event.dir: - if self.mmc.is_audio_file(event.name) and self.mmc.is_readable(full_filepath, False): + if self.mmc.is_audio_file(event.name) and self.mmc.is_readable(event.name, False): if event.cookie in self.temp_files: self.file_events.append({'filepath': event.pathname, 'mode': self.config.MODE_MODIFY}) del self.temp_files[event.cookie] From 495ad633336f61b4c22286ece683ed627d1464c4 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 17:37:30 -0400 Subject: [PATCH 13/29] CC-3732: NationTalk Database upgrade fail, from 2.0.3 to 2.1.0 -move NOT NULL constraint to after the column has been populated --- .../upgrades/airtime-2.1.0/common/Version20120411174904.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install_minimal/upgrades/airtime-2.1.0/common/Version20120411174904.php b/install_minimal/upgrades/airtime-2.1.0/common/Version20120411174904.php index 3c5891c99..436070b3b 100644 --- a/install_minimal/upgrades/airtime-2.1.0/common/Version20120411174904.php +++ b/install_minimal/upgrades/airtime-2.1.0/common/Version20120411174904.php @@ -13,18 +13,18 @@ class Version20120411174904 extends AbstractMigration public function up(Schema $schema) { $this->_addSql("ALTER TABLE cc_show_instances ADD created timestamp"); - $this->_addSql("ALTER TABLE cc_show_instances ALTER COLUMN created SET NOT NULL"); - $this->_addSql("ALTER TABLE cc_show_instances ADD last_scheduled timestamp"); //setting these to a default now for timeline refresh purposes. $now = gmdate("Y-m-d H:i:s"); $this->_addSql("UPDATE cc_show_instances SET created = '$now'"); $this->_addSql("UPDATE cc_show_instances SET last_scheduled = '$now'"); + + $this->_addSql("ALTER TABLE cc_show_instances ALTER COLUMN created SET NOT NULL"); } public function down(Schema $schema) { } -} \ No newline at end of file +} From b1611f5b863ae52277cccce053b459fdf01eca99 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 18:01:49 -0400 Subject: [PATCH 14/29] CC-3749: Media Monitor should not load files pypo cannot read -incorrect variable name --- .../media-monitor/airtimefilemonitor/airtimeprocessevent.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py index 14c5ec64d..d5986e868 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py @@ -236,10 +236,9 @@ class AirtimeProcessEvent(ProcessEvent): filename = self.mount_file_dir +"/mtab" if event.pathname in filename: self.handle_mount_change() - #if stuff dropped in stor via a UI move must change file permissions. - self.mmc.is_readable(event.pathname, event.dir) + if not event.dir: - if self.mmc.is_audio_file(event.name) and self.mmc.is_readable(event.name, False): + if self.mmc.is_audio_file(event.name) and self.mmc.is_readable(event.path, False): if event.cookie in self.temp_files: self.file_events.append({'filepath': event.pathname, 'mode': self.config.MODE_MODIFY}) del self.temp_files[event.cookie] From 0685148c107c0aee266c885bb7239f11be714195 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 18:05:02 -0400 Subject: [PATCH 15/29] CC-3749: Media Monitor should not load files pypo cannot read -incorrect variable name --- .../media-monitor/airtimefilemonitor/airtimeprocessevent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py index d5986e868..6d1ffbfa8 100644 --- a/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py +++ b/python_apps/media-monitor/airtimefilemonitor/airtimeprocessevent.py @@ -238,7 +238,7 @@ class AirtimeProcessEvent(ProcessEvent): self.handle_mount_change() if not event.dir: - if self.mmc.is_audio_file(event.name) and self.mmc.is_readable(event.path, False): + if self.mmc.is_audio_file(event.name) and self.mmc.is_readable(event.pathname, False): if event.cookie in self.temp_files: self.file_events.append({'filepath': event.pathname, 'mode': self.config.MODE_MODIFY}) del self.temp_files[event.cookie] From b01921abea9dc8dacf60f7409a26f94a5278b413 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 19:27:35 -0400 Subject: [PATCH 16/29] CC-3683: Upgraded database not equivalent to fresh install database part 1 -using raw sql statements generated by apgdiff --- .../upgrades/airtime-2.1.0/DbUpgrade.php | 16 ++- .../airtime-2.1.0/airtime-upgrade.php | 6 +- .../upgrades/airtime-2.1.0/data/upgrade.sql | 111 ++++++++++++++++++ 3 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 install_minimal/upgrades/airtime-2.1.0/data/upgrade.sql diff --git a/install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php b/install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php index cc999ccdf..c1e702f45 100644 --- a/install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php +++ b/install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php @@ -5,14 +5,22 @@ */ class AirtimeDatabaseUpgrade{ - public static function start(){ + public static function start($p_dbValues){ echo "* Updating Database".PHP_EOL; - self::task0(); + self::task0($p_dbValues); self::task1(); } - private static function task0(){ - UpgradeCommon::MigrateTablesToVersion(__DIR__, '20120411174904'); + private static function task0($p_dbValues){ + //UpgradeCommon::MigrateTablesToVersion(__DIR__, '20120411174904'); + + $username = $p_dbValues['database']['dbuser']; + $password = $p_dbValues['database']['dbpass']; + $host = $p_dbValues['database']['host']; + $database = $p_dbValues['database']['dbname']; + $dir = __DIR__; + + passthru("export PGPASSWORD=$password && psql -h $host -U $username -f $dir/data/upgrade.sql $database"); $sql = "INSERT INTO cc_pref(\"keystr\", \"valstr\") VALUES('scheduled_play_switch', 'on')"; UpgradeCommon::queryDb($sql); diff --git a/install_minimal/upgrades/airtime-2.1.0/airtime-upgrade.php b/install_minimal/upgrades/airtime-2.1.0/airtime-upgrade.php index 627e5ce2c..5265b0c6b 100644 --- a/install_minimal/upgrades/airtime-2.1.0/airtime-upgrade.php +++ b/install_minimal/upgrades/airtime-2.1.0/airtime-upgrade.php @@ -49,9 +49,13 @@ require_once 'ConfFileUpgrade.php'; require_once 'DbUpgrade.php'; require_once 'MiscUpgrade.php'; + +$filename = "/etc/airtime/airtime.conf"; +$values = parse_ini_file($filename, true); + UpgradeCommon::connectToDatabase(); UpgradeCommon::SetDefaultTimezone(); AirtimeConfigFileUpgrade::start(); -AirtimeDatabaseUpgrade::start(); +AirtimeDatabaseUpgrade::start($values); AirtimeMiscUpgrade::start(); diff --git a/install_minimal/upgrades/airtime-2.1.0/data/upgrade.sql b/install_minimal/upgrades/airtime-2.1.0/data/upgrade.sql new file mode 100644 index 000000000..0ae9a8c5b --- /dev/null +++ b/install_minimal/upgrades/airtime-2.1.0/data/upgrade.sql @@ -0,0 +1,111 @@ +DROP TRIGGER calculate_position ON cc_playlistcontents; + +DROP FUNCTION calculate_position(); + +DROP VIEW cc_playlisttimes; + +CREATE FUNCTION airtime_to_int(chartoconvert character varying) RETURNS integer + AS + 'SELECT CASE WHEN trim($1) SIMILAR TO ''[0-9]+'' THEN CAST(trim($1) AS integer) ELSE NULL END;' + LANGUAGE SQL + IMMUTABLE + RETURNS NULL ON NULL INPUT; + +ALTER TABLE cc_files + DROP CONSTRAINT cc_music_dirs_folder_fkey; + +ALTER TABLE cc_playlist + DROP CONSTRAINT cc_playlist_editedby_fkey; + +CREATE SEQUENCE cc_subjs_token_id_seq + START WITH 1 + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + +CREATE TABLE cc_subjs_token ( + id integer DEFAULT nextval('cc_subjs_token_id_seq'::regclass) NOT NULL, + user_id integer NOT NULL, + "action" character varying(255) NOT NULL, + token character varying(40) NOT NULL, + created timestamp without time zone NOT NULL +); + +ALTER TABLE cc_files + ADD COLUMN utime timestamp(6) without time zone, + ADD COLUMN lptime timestamp(6) without time zone, + ADD COLUMN file_exists boolean DEFAULT true, + ALTER COLUMN bit_rate TYPE integer USING airtime_to_int(bit_rate) /* TYPE change - table: cc_files original: character varying(32) new: integer */, + ALTER COLUMN sample_rate TYPE integer USING airtime_to_int(bit_rate) /* TYPE change - table: cc_files original: character varying(32) new: integer */, + ALTER COLUMN length TYPE interval /* TYPE change - table: cc_files original: time without time zone new: interval */, + ALTER COLUMN length SET DEFAULT '00:00:00'::interval; + +ALTER TABLE cc_music_dirs + ADD COLUMN "exists" boolean DEFAULT true, + ADD COLUMN watched boolean DEFAULT true; + +ALTER TABLE cc_playlist + DROP COLUMN "state", + DROP COLUMN currentlyaccessing, + DROP COLUMN editedby, + DROP COLUMN creator, + ADD COLUMN utime timestamp(6) without time zone, + ADD COLUMN creator_id integer, + ADD COLUMN length interval DEFAULT '00:00:00'::interval; + +ALTER TABLE cc_playlistcontents + ALTER COLUMN cliplength TYPE interval /* TYPE change - table: cc_playlistcontents original: time without time zone new: interval */, + ALTER COLUMN cliplength SET DEFAULT '00:00:00'::interval, + ALTER COLUMN cuein TYPE interval /* TYPE change - table: cc_playlistcontents original: time without time zone new: interval */, + ALTER COLUMN cuein SET DEFAULT '00:00:00'::interval, + ALTER COLUMN cueout TYPE interval /* TYPE change - table: cc_playlistcontents original: time without time zone new: interval */, + ALTER COLUMN cueout SET DEFAULT '00:00:00'::interval; + +ALTER TABLE cc_schedule + DROP COLUMN playlist_id, + DROP COLUMN group_id, + DROP COLUMN schedule_group_played, + ADD COLUMN playout_status smallint DEFAULT 1 NOT NULL, + ALTER COLUMN clip_length TYPE interval /* TYPE change - table: cc_schedule original: time without time zone new: interval */, + ALTER COLUMN clip_length SET DEFAULT '00:00:00'::interval, + ALTER COLUMN cue_in TYPE interval /* TYPE change - table: cc_schedule original: time without time zone new: interval */, + ALTER COLUMN cue_in SET DEFAULT '00:00:00'::interval, + ALTER COLUMN cue_out TYPE interval /* TYPE change - table: cc_schedule original: time without time zone new: interval */, + ALTER COLUMN cue_out SET DEFAULT '00:00:00'::interval; + +ALTER TABLE cc_show + ADD COLUMN live_stream_using_airtime_auth boolean DEFAULT false, + ADD COLUMN live_stream_using_custom_auth boolean DEFAULT false, + ADD COLUMN live_stream_user character varying(255), + ADD COLUMN live_stream_pass character varying(255); + +ALTER TABLE cc_show_instances + ADD COLUMN created timestamp without time zone, + ADD COLUMN last_scheduled timestamp without time zone, + ALTER COLUMN time_filled TYPE interval /* TYPE change - table: cc_show_instances original: time without time zone new: interval */, + ALTER COLUMN time_filled SET DEFAULT '00:00:00'::interval; + +UPDATE cc_show_instances SET created = now(); + +ALTER TABLE cc_show_instances + ALTER COLUMN created SET NOT NULL; + +ALTER TABLE cc_subjs_token + ADD CONSTRAINT cc_subjs_token_pkey PRIMARY KEY (id); + +ALTER TABLE cc_files + ADD CONSTRAINT cc_music_dirs_folder_fkey FOREIGN KEY (directory) REFERENCES cc_music_dirs(id); + +ALTER TABLE cc_playlist + ADD CONSTRAINT cc_playlist_createdby_fkey FOREIGN KEY (creator_id) REFERENCES cc_subjs(id); + +ALTER TABLE cc_subjs_token + ADD CONSTRAINT cc_subjs_token_idx UNIQUE (token); + +ALTER TABLE cc_subjs_token + ADD CONSTRAINT cc_subjs_token_userid_fkey FOREIGN KEY (user_id) REFERENCES cc_subjs(id) ON DELETE CASCADE; + +CREATE INDEX cc_files_file_exists_idx ON cc_files USING btree (file_exists); + +DROP FUNCTION airtime_to_int(chartoconvert character varying); From 240756f98bf71e771c9cf04e6f4d5178fa0a60f1 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 19:50:16 -0400 Subject: [PATCH 17/29] CC-3767: Pypo push crashed --- python_apps/pypo/pypopush.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python_apps/pypo/pypopush.py b/python_apps/pypo/pypopush.py index 6e8592c68..56736c107 100644 --- a/python_apps/pypo/pypopush.py +++ b/python_apps/pypo/pypopush.py @@ -129,7 +129,8 @@ class PypoPush(Thread): tn.write('exit\n') tn.read_all() except Exception, e: - self.logger.error(str(e)) + self.logger.error("Error connecting to Liquidsoap: %s", e) + response = [] finally: self.telnet_lock.release() From 9324d5d24afb609eecf0bf3aac23e7e3198ab0c5 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 19:51:08 -0400 Subject: [PATCH 18/29] CC-3683: Upgraded database not equivalent to fresh install database part 1 -using raw sql statements generated by apgdiff --- install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php b/install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php index c1e702f45..ab40c2afd 100644 --- a/install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php +++ b/install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php @@ -20,7 +20,7 @@ class AirtimeDatabaseUpgrade{ $database = $p_dbValues['database']['dbname']; $dir = __DIR__; - passthru("export PGPASSWORD=$password && psql -h $host -U $username -f $dir/data/upgrade.sql $database"); + passthru("export PGPASSWORD=$password && psql -h $host -U $username -f $dir/data/upgrade.sql $database > /dev/null"); $sql = "INSERT INTO cc_pref(\"keystr\", \"valstr\") VALUES('scheduled_play_switch', 'on')"; UpgradeCommon::queryDb($sql); From d80dece0bca3756135ac5399d3facdf2167661d6 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 19:56:20 -0400 Subject: [PATCH 19/29] CC-3683: Upgraded database not equivalent to fresh install database part 1 -hide useless "NOTICE" statements --- install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php b/install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php index ab40c2afd..1b78982fe 100644 --- a/install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php +++ b/install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php @@ -20,7 +20,7 @@ class AirtimeDatabaseUpgrade{ $database = $p_dbValues['database']['dbname']; $dir = __DIR__; - passthru("export PGPASSWORD=$password && psql -h $host -U $username -f $dir/data/upgrade.sql $database > /dev/null"); + passthru("export PGPASSWORD=$password && psql -h $host -U $username -f $dir/data/upgrade.sql $database > /dev/null | grep -v \"will create implicit index\""); $sql = "INSERT INTO cc_pref(\"keystr\", \"valstr\") VALUES('scheduled_play_switch', 'on')"; UpgradeCommon::queryDb($sql); From bc8cdc8a2d49b165474c750ae9922cbc7f7f318a Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 20:13:21 -0400 Subject: [PATCH 20/29] CC-3683: Upgraded database not equivalent to fresh install database part 1 -hide useless "NOTICE" statements --- install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php b/install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php index 1b78982fe..f2a5ab012 100644 --- a/install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php +++ b/install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php @@ -20,7 +20,7 @@ class AirtimeDatabaseUpgrade{ $database = $p_dbValues['database']['dbname']; $dir = __DIR__; - passthru("export PGPASSWORD=$password && psql -h $host -U $username -f $dir/data/upgrade.sql $database > /dev/null | grep -v \"will create implicit index\""); + passthru("export PGPASSWORD=$password && psql -h $host -U $username -q -f $dir/data/upgrade.sql $database"); $sql = "INSERT INTO cc_pref(\"keystr\", \"valstr\") VALUES('scheduled_play_switch', 'on')"; UpgradeCommon::queryDb($sql); From 5aea66a5c4be4768ad87ead9344bb35c6b9fe4ae Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Fri, 4 May 2012 20:18:11 -0400 Subject: [PATCH 21/29] CC-3683: Upgraded database not equivalent to fresh install database part 1 -hide useless "NOTICE" statements --- install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php b/install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php index f2a5ab012..8f2d006e1 100644 --- a/install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php +++ b/install_minimal/upgrades/airtime-2.1.0/DbUpgrade.php @@ -9,6 +9,7 @@ class AirtimeDatabaseUpgrade{ echo "* Updating Database".PHP_EOL; self::task0($p_dbValues); self::task1(); + echo " * Complete".PHP_EOL; } private static function task0($p_dbValues){ @@ -20,7 +21,7 @@ class AirtimeDatabaseUpgrade{ $database = $p_dbValues['database']['dbname']; $dir = __DIR__; - passthru("export PGPASSWORD=$password && psql -h $host -U $username -q -f $dir/data/upgrade.sql $database"); + passthru("export PGPASSWORD=$password && psql -h $host -U $username -q -f $dir/data/upgrade.sql $database 2>&1 | grep -v \"will create implicit index\""); $sql = "INSERT INTO cc_pref(\"keystr\", \"valstr\") VALUES('scheduled_play_switch', 'on')"; UpgradeCommon::queryDb($sql); From bb5deda09d12fd227303acd43d63ad2f022f606f Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Sat, 5 May 2012 21:34:09 -0400 Subject: [PATCH 22/29] Various fixes for installer -create zendphp.log before web files are available -run airtime.php script earlier so DB is created before web files are available -zendphp.log should have 644 permissions, not 755 --- install_minimal/airtime-install | 13 +++++++++++++ install_minimal/include/AirtimeInstall.php | 4 ++-- install_minimal/include/airtime-copy-files.sh | 16 +++++++++++++++- install_minimal/include/airtime-initialize.sh | 16 ---------------- install_minimal/include/airtime-install.php | 7 +------ 5 files changed, 31 insertions(+), 25 deletions(-) diff --git a/install_minimal/airtime-install b/install_minimal/airtime-install index 70efa614e..da7663a5f 100755 --- a/install_minimal/airtime-install +++ b/install_minimal/airtime-install @@ -160,6 +160,19 @@ if [ "$DO_UPGRADE" -eq "1" ]; then php --php-ini ${SCRIPTPATH}/airtime-php.ini ${SCRIPTPATH}/include/airtime-upgrade.php $@ fi +set +e +if [ "$DO_UPGRADE" -eq "0" ]; then + php --php-ini ${SCRIPTPATH}/airtime-php.ini ${SCRIPTPATH}/include/airtime-install.php $@ + result=$? + + if [ "$result" -ne "0" ]; then + #There was an error, exit with error code. + echo "There was an error during install. Exit code $result" + exit 1 + fi +fi +set -e + $SCRIPTPATH/include/airtime-copy-files.sh $SCRIPTPATH/include/airtime-initialize.sh $@ diff --git a/install_minimal/include/AirtimeInstall.php b/install_minimal/include/AirtimeInstall.php index 4dc8555a8..7186c0833 100644 --- a/install_minimal/include/AirtimeInstall.php +++ b/install_minimal/include/AirtimeInstall.php @@ -261,7 +261,7 @@ class AirtimeInstall // Put Propel sql files in Database //$command = AirtimeInstall::CONF_DIR_WWW."/library/propel/generator/bin/propel-gen ".AirtimeInstall::CONF_DIR_WWW."/build/ insert-sql 2>/dev/null"; - $dir = AirtimeInstall::CONF_DIR_WWW."/build/sql/"; + $dir = self::GetAirtimeSrcDir()."/build/sql/"; $files = array("schema.sql", "sequences.sql", "views.sql", "triggers.sql", "defaultdata.sql"); foreach ($files as $f){ @@ -453,7 +453,7 @@ class AirtimeInstall } touch($file); - chmod($file, 0755); + chmod($file, 0644); chown($file, $CC_CONFIG['webServerUser']); chgrp($file, $CC_CONFIG['webServerUser']); } diff --git a/install_minimal/include/airtime-copy-files.sh b/install_minimal/include/airtime-copy-files.sh index b46b1d621..236cf5f98 100755 --- a/install_minimal/include/airtime-copy-files.sh +++ b/install_minimal/include/airtime-copy-files.sh @@ -35,11 +35,18 @@ AIRTIMEROOT=$SCRIPTPATH/../../ echo "* Creating /etc/airtime" mkdir -p /etc/airtime -#if [ "$DO_UPGRADE" -eq "0" ]; then if [ ! -e /etc/airtime/airtime.conf ]; then + #config file airtime.conf exists, but Airtime is not installed cp $AIRTIMEROOT/airtime_mvc/build/airtime.conf /etc/airtime fi +#if [ -e /etc/airtime/airtime.conf -a "$DO_UPGRADE" -eq "0" ]; then + #config file airtime.conf exists, but Airtime is not installed +# mv /etc/airtime/airtime.conf airtime.conf.bak +# cp $AIRTIMEROOT/airtime_mvc/build/airtime.conf /etc/airtime +#fi + + echo "* Creating /etc/monit/conf.d/monit-airtime-generic.cfg" mkdir -p /etc/monit/conf.d/ if [ ! -e /etc/monit/conf.d/monit-airtime-generic.cfg ]; then @@ -78,6 +85,13 @@ ln -sf /usr/lib/airtime/utils/airtime-log /usr/bin/airtime-log ln -sf /usr/lib/airtime/utils/airtime-test-soundcard /usr/bin/airtime-test-soundcard ln -sf /usr/lib/airtime/utils/airtime-test-stream /usr/bin/airtime-test-stream +echo "* Creating /var/log/airtime" +mkdir -p /var/log/airtime +chmod a+x /var/log/airtime +touch /var/log/airtime/zendphp.log +chown www-data:www-data /var/log/airtime/zendphp.log +chmod 644 /var/log/airtime/zendphp.log + if [ "$web" = "t" ]; then echo "* Creating /usr/share/airtime" rm -rf "/usr/share/airtime" diff --git a/install_minimal/include/airtime-initialize.sh b/install_minimal/include/airtime-initialize.sh index d3d6cdfed..761f09cc3 100755 --- a/install_minimal/include/airtime-initialize.sh +++ b/install_minimal/include/airtime-initialize.sh @@ -14,22 +14,6 @@ SCRIPTPATH=`dirname $SCRIPT` AIRTIMEROOT=$SCRIPTPATH/../../ -#virtualenv_bin="/usr/lib/airtime/airtime_virtualenv/bin/" -#. ${virtualenv_bin}activate - -set +e -if [ "$DO_UPGRADE" -eq "0" ]; then - php --php-ini ${SCRIPTPATH}/../airtime-php.ini ${SCRIPTPATH}/airtime-install.php $@ - result=$? - - if [ "$result" -ne "0" ]; then - #There was an error, exit with error code. - echo "There was an error during install. Exit code $result" - exit 1 - fi -fi -set -e - if [ "$mediamonitor" = "t" ]; then python $AIRTIMEROOT/python_apps/media-monitor/install/media-monitor-initialize.py fi diff --git a/install_minimal/include/airtime-install.php b/install_minimal/include/airtime-install.php index 6ff0aa492..a17937348 100644 --- a/install_minimal/include/airtime-install.php +++ b/install_minimal/include/airtime-install.php @@ -3,9 +3,6 @@ * @package Airtime * @copyright 2011 Sourcefabric O.P.S. * @license http://www.gnu.org/licenses/gpl.txt - * - * Checks if a previous version of Airtime is currently installed and upgrades Airtime if so. - * Performs a new install (new configs, database install) otherwise. */ require_once(__DIR__.'/AirtimeIni.php'); require_once(__DIR__.'/AirtimeInstall.php'); @@ -56,7 +53,7 @@ if ($overwrite) { } // Update the build.properties file to point to the correct directory. -AirtimeIni::UpdateIniValue(AirtimeInstall::CONF_DIR_WWW.'/build/build.properties', 'project.home', AirtimeInstall::CONF_DIR_WWW); +//AirtimeIni::UpdateIniValue(AirtimeInstall::CONF_DIR_WWW.'/build/build.properties', 'project.home', AirtimeInstall::CONF_DIR_WWW); require_once(AirtimeInstall::GetAirtimeSrcDir().'/application/configs/conf.php'); @@ -74,6 +71,4 @@ if ($db_install) { } } -AirtimeInstall::CreateZendPhpLogFile(); - /* FINISHED AIRTIME PHP INSTALLER */ From 5204da45021ed54cf0fb91cbb15e0af2b03bee05 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Sat, 5 May 2012 22:28:52 -0400 Subject: [PATCH 23/29] -new logging class -don't log errors to stdout -log all errors to /var/log/airtime/zendphp.log --- .../application/logging/AirtimeLog.php | 109 ++++++++++++++++++ airtime_mvc/application/logging/Logging.php | 13 ++- airtime_mvc/public/index.php | 4 +- 3 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 airtime_mvc/application/logging/AirtimeLog.php diff --git a/airtime_mvc/application/logging/AirtimeLog.php b/airtime_mvc/application/logging/AirtimeLog.php new file mode 100644 index 000000000..6691f51e9 --- /dev/null +++ b/airtime_mvc/application/logging/AirtimeLog.php @@ -0,0 +1,109 @@ +_errorHandlerMap[$errno])) { + $priority = $this->_errorHandlerMap[$errno]; + } else { + $priority = Zend_Log::INFO; + } + $this->log($errstr, $priority, array('errno'=>$errno, 'file'=>$errfile, 'line'=>$errline, 'context'=>$errcontext)); + } + + if ($this->_origErrorHandler !== null) { + return call_user_func($this->_origErrorHandler, $errno, $errstr, $errfile, $errline, $errcontext); + } + return false; + } + + /** + * Register Logging system as an error handler to log php errors + * Note: it still calls the original error handler if set_error_handler is able to return it. + * + * Errors will be mapped as: + * E_NOTICE, E_USER_NOTICE => NOTICE + * E_WARNING, E_CORE_WARNING, E_USER_WARNING => WARN + * E_ERROR, E_USER_ERROR, E_CORE_ERROR, E_RECOVERABLE_ERROR => ERR + * E_DEPRECATED, E_STRICT, E_USER_DEPRECATED => DEBUG + * (unknown/other) => INFO + * + * @link http://www.php.net/manual/en/function.set-error-handler.php Custom error handler + * + * @return Zend_Log + */ + public function registerErrorHandler() + { + // Only register once. Avoids loop issues if it gets registered twice. + if ($this->_registeredErrorHandler) { + return $this; + } + + $this->_origErrorHandler = set_error_handler(array($this, 'errorHandler')); + + // Contruct a default map of phpErrors to Zend_Log priorities. + // Some of the errors are uncatchable, but are included for completeness + $this->_errorHandlerMap = array( + E_NOTICE => Zend_Log::NOTICE, + E_USER_NOTICE => Zend_Log::NOTICE, + E_WARNING => Zend_Log::WARN, + E_CORE_WARNING => Zend_Log::WARN, + E_USER_WARNING => Zend_Log::WARN, + E_ERROR => Zend_Log::ERR, + E_USER_ERROR => Zend_Log::ERR, + E_CORE_ERROR => Zend_Log::ERR, + E_RECOVERABLE_ERROR => Zend_Log::ERR, + E_STRICT => Zend_Log::DEBUG, + ); + // PHP 5.3.0+ + if (defined('E_DEPRECATED')) { + $this->_errorHandlerMap['E_DEPRECATED'] = Zend_Log::DEBUG; + } + if (defined('E_USER_DEPRECATED')) { + $this->_errorHandlerMap['E_USER_DEPRECATED'] = Zend_Log::DEBUG; + } + + $this->_registeredErrorHandler = true; + return $this; + } +} diff --git a/airtime_mvc/application/logging/Logging.php b/airtime_mvc/application/logging/Logging.php index 5c25b831f..50e6b610e 100644 --- a/airtime_mvc/application/logging/Logging.php +++ b/airtime_mvc/application/logging/Logging.php @@ -6,9 +6,18 @@ class Logging { private static $_path; public static function getLogger(){ - if (!isset(self::$logger)) { + if (!isset(self::$_logger)) { $writer = new Zend_Log_Writer_Stream(self::$_path); - self::$_logger = new Zend_Log($writer); + + if (Zend_Version::compareVersion("1.11") > 0){ + //Running Zend version 1.10 or lower. Need to instantiate our + //own Zend Log class with backported code from 1.11. + require_once __DIR__."/AirtimeLog.php"; + self::$_logger = new Airtime_Zend_Log($writer); + } else { + self::$_logger = new Zend_Log($writer); + } + self::$_logger->registerErrorHandler(); } return self::$_logger; } diff --git a/airtime_mvc/public/index.php b/airtime_mvc/public/index.php index f4fc46922..e496f133c 100644 --- a/airtime_mvc/public/index.php +++ b/airtime_mvc/public/index.php @@ -1,8 +1,8 @@ Date: Sat, 5 May 2012 22:29:16 -0400 Subject: [PATCH 24/29] -catch all exceptions in Preference model and make header 503 --- airtime_mvc/application/models/Preference.php | 195 +++++++++--------- 1 file changed, 102 insertions(+), 93 deletions(-) diff --git a/airtime_mvc/application/models/Preference.php b/airtime_mvc/application/models/Preference.php index 90e87b9b2..f0341e879 100644 --- a/airtime_mvc/application/models/Preference.php +++ b/airtime_mvc/application/models/Preference.php @@ -3,102 +3,109 @@ class Application_Model_Preference { - public static function SetValue($key, $value, $isUserValue = false){ - global $CC_CONFIG; - $con = Propel::getConnection(); - - //called from a daemon process - if(!class_exists("Zend_Auth", false) || !Zend_Auth::getInstance()->hasIdentity()) { - $id = NULL; - } - else { - $auth = Zend_Auth::getInstance(); - $id = $auth->getIdentity()->id; - } - - $key = pg_escape_string($key); - $value = pg_escape_string($value); - - //Check if key already exists - $sql = "SELECT COUNT(*) FROM cc_pref" - ." WHERE keystr = '$key'"; - - //For user specific preference, check if id matches as well - if($isUserValue) { - $sql .= " AND subjid = '$id'"; - } - - $result = $con->query($sql)->fetchColumn(0); - - if($value == "") { - $value = "NULL"; - }else { - $value = "'$value'"; - } - - if($result == 1) { - // result found - if(is_null($id) || !$isUserValue) { - // system pref - $sql = "UPDATE cc_pref" - ." SET subjid = NULL, valstr = $value" - ." WHERE keystr = '$key'"; - } else { - // user pref - $sql = "UPDATE cc_pref" - . " SET valstr = $value" - . " WHERE keystr = '$key' AND subjid = $id"; - } - } else { - // result not found - if(is_null($id) || !$isUserValue) { - // system pref - $sql = "INSERT INTO cc_pref (keystr, valstr)" - ." VALUES ('$key', $value)"; - } else { - // user pref - $sql = "INSERT INTO cc_pref (subjid, keystr, valstr)" - ." VALUES ($id, '$key', $value)"; - } - } + public static function SetValue($key, $value, $isUserValue = false){ try { - $con->exec($sql); - } catch (Exception $e){ - Logging::log("Could not connect to database."); - header('HTTP/1.0 503 Service Unavailable'); - exit; - } - } + $con = Propel::getConnection(); - public static function GetValue($key, $isUserValue = false){ - global $CC_CONFIG; - $con = Propel::getConnection(); + //called from a daemon process + if(!class_exists("Zend_Auth", false) || !Zend_Auth::getInstance()->hasIdentity()) { + $id = NULL; + } + else { + $auth = Zend_Auth::getInstance(); + $id = $auth->getIdentity()->id; + } - //Check if key already exists - $sql = "SELECT COUNT(*) FROM cc_pref" - ." WHERE keystr = '$key'"; - //For user specific preference, check if id matches as well - if ($isUserValue) { - $auth = Zend_Auth::getInstance(); - if($auth->hasIdentity()) { - $id = $auth->getIdentity()->id; - $sql .= " AND subjid = '$id'"; - } - } - $result = $con->query($sql)->fetchColumn(0); - if ($result == 0) - return ""; - else { - $sql = "SELECT valstr FROM cc_pref" + $key = pg_escape_string($key); + $value = pg_escape_string($value); + + //Check if key already exists + $sql = "SELECT COUNT(*) FROM cc_pref" ." WHERE keystr = '$key'"; - //For user specific preference, check if id matches as well - if($isUserValue && $auth->hasIdentity()) { - $sql .= " AND subjid = '$id'"; - } - + //For user specific preference, check if id matches as well + if($isUserValue) { + $sql .= " AND subjid = '$id'"; + } + $result = $con->query($sql)->fetchColumn(0); - return ($result !== false) ? $result : ""; + + if($value == "") { + $value = "NULL"; + }else { + $value = "'$value'"; + } + + if($result == 1) { + // result found + if(is_null($id) || !$isUserValue) { + // system pref + $sql = "UPDATE cc_pref" + ." SET subjid = NULL, valstr = $value" + ." WHERE keystr = '$key'"; + } else { + // user pref + $sql = "UPDATE cc_pref" + . " SET valstr = $value" + . " WHERE keystr = '$key' AND subjid = $id"; + } + } else { + // result not found + if(is_null($id) || !$isUserValue) { + // system pref + $sql = "INSERT INTO cc_pref (keystr, valstr)" + ." VALUES ('$key', $value)"; + } else { + // user pref + $sql = "INSERT INTO cc_pref (subjid, keystr, valstr)" + ." VALUES ($id, '$key', $value)"; + } + } + + $con->exec($sql); + + } catch (Exception $e){ + header('HTTP/1.0 503 Service Unavailable'); + Logging::log("Could not connect to database."); + exit; + } + + } + + public static function GetValue($key, $isUserValue = false){ + try { + $con = Propel::getConnection(); + + //Check if key already exists + $sql = "SELECT COUNT(*) FROM cc_pref" + ." WHERE keystr = '$key'"; + //For user specific preference, check if id matches as well + if ($isUserValue) { + $auth = Zend_Auth::getInstance(); + if($auth->hasIdentity()) { + $id = $auth->getIdentity()->id; + $sql .= " AND subjid = '$id'"; + } + } + $result = $con->query($sql)->fetchColumn(0); + if ($result == 0) + return ""; + else { + $sql = "SELECT valstr FROM cc_pref" + ." WHERE keystr = '$key'"; + + //For user specific preference, check if id matches as well + if($isUserValue && $auth->hasIdentity()) { + $sql .= " AND subjid = '$id'"; + } + + $result = $con->query($sql)->fetchColumn(0); + return ($result !== false) ? $result : ""; + } + } catch (Exception $e) { + header('HTTP/1.0 503 Service Unavailable'); + Logging::log("Could not connect to database."); + exit; } } @@ -567,10 +574,12 @@ class Application_Model_Preference public static function GetAirtimeVersion(){ if (defined('APPLICATION_ENV') && APPLICATION_ENV == "development" && function_exists('exec')){ - return self::GetValue("system_version")."+".exec("git rev-parse --short HEAD"); - } else { - return self::GetValue("system_version"); + $version = exec("git rev-parse --short HEAD 2>/dev/null", $out, $return_code); + if ($return_code == 0){ + return self::GetValue("system_version")."+".$version; + } } + return self::GetValue("system_version"); } public static function GetLatestVersion(){ From adee58bcbcaf3c1a5e82c3626388a48420e81bb4 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Sat, 5 May 2012 22:29:31 -0400 Subject: [PATCH 25/29] -better handling of timezones --- airtime_mvc/application/Bootstrap.php | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/airtime_mvc/application/Bootstrap.php b/airtime_mvc/application/Bootstrap.php index 1e5fc3b2d..fe5634682 100644 --- a/airtime_mvc/application/Bootstrap.php +++ b/airtime_mvc/application/Bootstrap.php @@ -12,21 +12,26 @@ require_once "DateHelper.php"; require_once "OsPath.php"; require_once __DIR__.'/controllers/plugins/RabbitMqPlugin.php'; + +//DateTime in PHP 5.3.0+ need a default timezone set. Set to UTC initially +//in case Application_Model_Preference::GetTimezone fails and creates needs to create +//a log entry. This log entry requires a call to date(), which then complains that +//timezone isn't set. Setting a default timezone allows us to create a a graceful log +//that getting the real timezone failed, without PHP complaining that it cannot log because +//there is no timezone :|. +date_default_timezone_set('UTC'); +date_default_timezone_set(Application_Model_Preference::GetTimezone()); + global $CC_CONFIG; $CC_CONFIG['airtime_version'] = Application_Model_Preference::GetAirtimeVersion(); require_once __DIR__."/configs/navigation.php"; -//DateTime in PHP 5.3.0+ need a default timezone set. -date_default_timezone_set(Application_Model_Preference::GetTimezone()); - Zend_Validate::setDefaultNamespaces("Zend"); $front = Zend_Controller_Front::getInstance(); $front->registerPlugin(new RabbitMqPlugin()); -//Logging::debug($_SERVER['REQUEST_URI']); - /* The bootstrap class should only be used to initialize actions that return a view. Actions that return JSON will not use the bootstrap class! */ class Bootstrap extends Zend_Application_Bootstrap_Bootstrap From 4ad3286db274bb5842e3560efa44d1e5c5f760bf Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Sun, 6 May 2012 15:34:41 -0400 Subject: [PATCH 26/29] CC-3769: PHP 5.3.2 issue with DateTime::createFromFormat and format string "U.u" --- airtime_mvc/application/models/Scheduler.php | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/airtime_mvc/application/models/Scheduler.php b/airtime_mvc/application/models/Scheduler.php index 364e24cc6..744d65c9e 100644 --- a/airtime_mvc/application/models/Scheduler.php +++ b/airtime_mvc/application/models/Scheduler.php @@ -25,6 +25,12 @@ class Application_Model_Scheduler { $this->epochNow = microtime(true); $this->nowDT = DateTime::createFromFormat("U.u", $this->epochNow, new DateTimeZone("UTC")); + + if ($this->nowDT === false){ + // DateTime::createFromFormat does not support millisecond string formatting in PHP 5.3.2 (Ubuntu 10.04). + // In PHP 5.3.3 (Ubuntu 10.10), this has been fixed. + $this->nowDT = DateTime::createFromFormat("U", time(), new DateTimeZone("UTC")); + } $this->user = Application_Model_User::GetCurrentUser(); } @@ -190,6 +196,12 @@ class Application_Model_Scheduler { $endEpoch = bcadd($startEpoch , (string) $durationSeconds, 6); $dt = DateTime::createFromFormat("U.u", $endEpoch, new DateTimeZone("UTC")); + + if ($dt === false) { + //PHP 5.3.2 problem + $dt = DateTime::createFromFormat("U", intval($endEpoch), new DateTimeZone("UTC")); + } + return $dt; } From 6135660d0f1b42c0cbc1590214421bb64c017b4a Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Sun, 6 May 2012 20:05:14 -0400 Subject: [PATCH 27/29] CC-3771: year column in cc_files sometimes has values greater than 2^31-1, which causes postgresql to fail when trying to convert this to a year. --- airtime_mvc/application/models/StoredFile.php | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/airtime_mvc/application/models/StoredFile.php b/airtime_mvc/application/models/StoredFile.php index a6044c868..bb3bfed0b 100644 --- a/airtime_mvc/application/models/StoredFile.php +++ b/airtime_mvc/application/models/StoredFile.php @@ -91,6 +91,27 @@ class Application_Model_StoredFile { } else { $dbMd = array(); + + if (isset($p_md["year"])){ + // We need to make sure to clean this value before inserting into database. + // If value is outside of range [-2^31, 2^31-1] then postgresl will throw error + // when trying to retrieve this value. We could make sure number is within these bounds, + // but simplest is to do substring to 4 digits (both values are garbage, but at least our + // new garbage value won't cause errors). If the value is 2012-01-01, then substring to + // 4 digits is an OK result. + // CC-3771 + + $year = $p_md["year"]; + + if (strlen($year) > 4){ + $year = substr($year, 0, 4); + } + if (!is_numeric($year)){ + $year = 0; + } + $p_md["year"] = $year; + } + foreach ($p_md as $mdConst => $mdValue) { $dbMd[constant($mdConst)] = $mdValue; } From 77bffcd8e9c21c881c2d3b669271c7386ce666f5 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Sun, 6 May 2012 20:30:49 -0400 Subject: [PATCH 28/29] join function should be static --- airtime_mvc/application/common/OsPath.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airtime_mvc/application/common/OsPath.php b/airtime_mvc/application/common/OsPath.php index 957545dc4..d9d9b6eff 100644 --- a/airtime_mvc/application/common/OsPath.php +++ b/airtime_mvc/application/common/OsPath.php @@ -45,7 +45,7 @@ class Application_Common_OsPath{ /* Similar to the os.path.join python method * http://stackoverflow.com/a/1782990/276949 */ - function join() { + public static function join() { $args = func_get_args(); $paths = array(); From bd6686a24348d343fde3dd798fc1dca946b84562 Mon Sep 17 00:00:00 2001 From: Martin Konecny Date: Sun, 6 May 2012 20:45:52 -0400 Subject: [PATCH 29/29] CC-3771: year column in cc_files sometimes has values greater than 2^31-1, which causes postgresql to fail when trying to convert this to a year. fixed --- airtime_mvc/application/models/StoredFile.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/airtime_mvc/application/models/StoredFile.php b/airtime_mvc/application/models/StoredFile.php index bb3bfed0b..d699b47d1 100644 --- a/airtime_mvc/application/models/StoredFile.php +++ b/airtime_mvc/application/models/StoredFile.php @@ -92,7 +92,7 @@ class Application_Model_StoredFile { else { $dbMd = array(); - if (isset($p_md["year"])){ + if (isset($p_md["MDATA_KEY_YEAR"])){ // We need to make sure to clean this value before inserting into database. // If value is outside of range [-2^31, 2^31-1] then postgresl will throw error // when trying to retrieve this value. We could make sure number is within these bounds, @@ -101,7 +101,7 @@ class Application_Model_StoredFile { // 4 digits is an OK result. // CC-3771 - $year = $p_md["year"]; + $year = $p_md["MDATA_KEY_YEAR"]; if (strlen($year) > 4){ $year = substr($year, 0, 4); @@ -109,9 +109,9 @@ class Application_Model_StoredFile { if (!is_numeric($year)){ $year = 0; } - $p_md["year"] = $year; + $p_md["MDATA_KEY_YEAR"] = $year; } - + foreach ($p_md as $mdConst => $mdValue) { $dbMd[constant($mdConst)] = $mdValue; }