weird things

coolamqp/attaches/publisher.py

@@ -145,7 +145,11 @@ class Publisher(Channeler, Synchronized):
         assert self.mode == Publisher.MODE_CNPUB
 
         while len(self.messages) > 0:
-            msg, xchg, rk, fut = self.messages.popleft()
+            try:
+                msg, xchg, rk, fut = self.messages.popleft()
+            except IndexError:
+                #todo see docs/casefile-0001
+                break
 
             if not fut.set_running_or_notify_cancel():
                 continue  # cancelled

docs/casefile-0001

+Exception caught on 2017-02-25T20:29:08.025256:
+Traceback (most recent call last):
+  File "/smok4/src/sai/basic/__init__.py", line 71, in run_with_except_hook
+    run_old(*args, **kw)
+  File "/smok4/src/vesta/initiator/__init__.py", line 221, in run
+    log(**contents)
+  File "/smok4/src/sai/basic/logmaker/__init__.py", line 34, in log
+    handle.options(tx=False)(kwargs)
+  File "/usr/local/lib/python2.7/dist-packages/loveseal/task.py", line 139, in roll
+    r = self.region.amqp.publish(msg, routing_key=self.region.deadletter_queue, tx=True)
+  File "/smok4/src/sai/basic/msgs.py", line 83, in publish
+    return super(_AMQP, self).publish(msg, exchange, routing_key, tx)
+  File "/usr/local/lib/python2.7/dist-packages/coolamqp/clustering/cluster.py", line 145, in publish
+    return (self.pub_tr if tx else self.pub_na).publish(message, exchange, routing_key)
+  File "/usr/local/lib/python2.7/dist-packages/coolamqp/attaches/utils.py", line 230, in monitored
+    return fun(*args, **kwargs)
+  File "/usr/local/lib/python2.7/dist-packages/coolamqp/attaches/publisher.py", line 214, in publish
+    self._mode_cnpub_process_deliveries()
+  File "/usr/local/lib/python2.7/dist-packages/coolamqp/attaches/publisher.py", line 148, in _mode_cnpub_process_deliveries
+    msg, xchg, rk, fut = self.messages.popleft()
+IndexError: pop from an empty deque
+
+* Stack trace, innermost first
+** _mode_cnpub_process_deliveries at /usr/local/lib/python2.7/dist-packages/coolamqp/attaches/publisher.py:148
+*** self: <coolamqp.attaches.publisher.Publisher object at 0x7f6267dca810>
+** publish at /usr/local/lib/python2.7/dist-packages/coolamqp/attaches/publisher.py:214
+*** fut: <Future at 0x7f623eeab2d0 state=running>
+*** cnpo: CnpubMessageSendOrder(message=<coolamqp.objects.Message object at 0x7f623eeabdd0>, exchange_name='', routing_key='loveseal-deadletter', future=<Future at 0x7f623eeab2d0 state=running>)
+*** exchange: ''
+*** self: <coolamqp.attaches.publisher.Publisher object at 0x7f6267dca810>
+*** routing_key: 'loveseal-deadletter'
+*** message: <coolamqp.objects.Message object at 0x7f623eeabdd0>
+** monitored at /usr/local/lib/python2.7/dist-packages/coolamqp/attaches/utils.py:230
+*** fun: <function publish at 0x7f626aaea578>
+*** args: (<coolamqp.attaches.publisher.Publisher object at 0x7f6267dca810>, <coolamqp.objects.Message object at 0x7f623eeabdd0>, '', 'loveseal-deadletter')
+*** kwargs: {}
+** publish at /usr/local/lib/python2.7/dist-packages/coolamqp/clustering/cluster.py:147
+*** tx: True
+*** confirm: None
+*** self: <sai.basic.msgs._AMQP object at 0x7f626a841190>
+*** exchange: ''
+*** routing_key: 'loveseal-deadletter'
+*** message: <coolamqp.objects.Message object at 0x7f623eeabdd0>
+** publish at /smok4/src/sai/basic/msgs.py:83
+*** tx: True
+*** exchange: ''
+*** self: <sai.basic.msgs._AMQP object at 0x7f626a841190>
+*** routing_key: 'loveseal-deadletter'
+*** content: <coolamqp.objects.Message object at 0x7f623eeabdd0>
+*** msg: <coolamqp.objects.Message object at 0x7f623eeabdd0>
+** roll at /usr/local/lib/python2.7/dist-packages/loveseal/task.py:139
+*** body: '\x80\x02}q\x01(U\x06regionq\x02U\nsmok4-coreq\x03U\x04argsq\x04}q\x05(U\x0cping_profileq\x06K\x00U\x0benc_profileq\x07K\x01U\x07journalq\x08U\rsecurity/ssspq\tU\x05labelq\nU\x11sssp.secexceptionq\x0bU\x07sssp_idq\x0cJ\x01\x00\x17\x01U\ttimestampq\rGA\xd6,v\xc0\xfe\xb75U\x07messageq\x0eU\x1aSSSP device does not existq\x0fU\x06loggerq\x10U\x0evesta2408@dev1U\tremote_ipq\x11U\x10172.17.0.1:35779q\x12u\x85q\x13U\x02idq\x14U 4743bd5a817d4c23baaf3fe25b8aa4f0q\x15U\x04nameq\x16U\x1esai.basic.logmaker.lvsl.handleq\x17u.'
+*** job_id: '4743bd5a817d4c23baaf3fe25b8aa4f0'
+*** tx: False
+*** opt_kwargs: {'tx': False}
+*** self: <loveseal.task.Task object at 0x7f623ff03750>
+*** args: ({'ping_profile': 0, 'enc_profile': 1, 'journal': 'security/sssp', 'label': 'sssp.secexception', 'sssp_id': 18284545, 'timestamp': 1488050947.979932, 'message': 'SSSP device does not exist', 'logger': 'vesta2408@dev1', 'remote_ip': '172.17.0.1:35779'},)
+*** job: <loveseal.region.job.NormalJob object at 0x7f623ff12850>
+*** job_to_dict: <function job_to_dict at 0x7f623ff20410>
+*** NormalJob: <class 'loveseal.region.job.NormalJob'>
+*** kwargs: {}
+*** msg: <coolamqp.objects.Message object at 0x7f623eeabdd0>
+*** props: <coolamqp.framing.compilation.content_property.ParticularContentTypeList object at 0x7f6267cd31b8>
+** log at /smok4/src/sai/basic/logmaker/__init__.py:34
+*** Configuration: <class 'sai.basic.config.Configuration'>
+*** handle: <loveseal.task.Task object at 0x7f623ff03750>
+*** kwargs: {'ping_profile': 0, 'enc_profile': 1, 'journal': 'security/sssp', 'label': 'sssp.secexception', 'sssp_id': 18284545, 'timestamp': 1488050947.979932, 'message': 'SSSP device does not exist', 'logger': 'vesta2408@dev1', 'remote_ip': '172.17.0.1:35779'}
+** run at /smok4/src/vesta/initiator/__init__.py:221
+*** self: <SSSPInitiator(Thread-19, started 140060586936064)>
+*** e: SecException()
+*** contents: {'sssp_id': 18284545, 'ping_profile': 0, 'enc_profile': 1, 'message': 'SSSP device does not exist', 'label': 'sssp.secexception', 'journal': 'security/sssp', 'remote_ip': '172.17.0.1:35779'}
+** run_with_except_hook at /smok4/src/sai/basic/__init__.py:73
+*** args: ()
+*** kw: {}
+*** run_old: <bound method SSSPInitiator.run of <SSSPInitiator(Thread-19, started 140060586936064)>>
+** __bootstrap_inner at /usr/lib/python2.7/threading.py:810
+*** self: <SSSPInitiator(Thread-19, started 140060586936064)>
+** __bootstrap at /usr/lib/python2.7/threading.py:783
+*** self: <SSSPInitiator(Thread-19, started 140060586936064)>
+*************************************************