
oTc           @   s  d  d l  Z  d  d l Z d  d l Z d  d l m Z m Z d  d l m Z d  d l Z d  d l	 Z	 d  d l
 Z
 d  d l m Z m Z m Z d  d l m Z d  d l m Z m Z d  d l j Z d  d l Td  d l j Z d  d l j Z d  d l m Z d  d l Z
 d	 Z d
 Z d Z  d Z! d Z" d Z# d Z$ d Z% e   Z& d e' f d     YZ( d e f d     YZ) d   Z* d   Z+ d   Z, d   Z- d   Z. d S(   iN(   t   Threadt   RLock(   t   NNTPPermanentError(   t   synchronizedt   synchronized_CVt   CV(   t   Decoder(   t   NewsWrappert   request_server_info(   t   *(   t   BPSMeteri   i   i
   i   g?t   Serverc           B   s>   e  Z d d e d  d  Z e d    Z d   Z d   Z RS(   i    c         C   s  | |  _  d  |  _ t |  _ | |  _ | |  _ | |  _ | |  _ | |  _	 | |  _
 |
 |  _ | |  _ | |  _ |	 |  _ g  |  _ g  |  _ t |  _ d |  _ d |  _ d |  _ d  |  _ t |  _ d | k |  _ t |  _ x1 t |  D]# } |  j j t |  | d   q Wd  S(   Ni    t    s   free.xsusenet.comi   (   t   idt   Nonet   newidt   Falset   restartt   hostt   portt   timeoutt   threadst
   fillservert   sslt   optionalt	   retentiont   usernamet   passwordt   busy_threadst   idle_threadst   Truet   activet   bad_const   errormsgt   warningt   infot   requestt	   have_bodyt	   have_statt   ranget   appendR   (   t   selfR   R   R   R   R   R   R   R   R   R   R   t   i(    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   __init__9   s2    																						c         C   s   t  j   r{ |  j r{ t |  j  d k r{ t j d t |  j  d  } |  j | d d } t j d |  j | f  n	 |  j } | S(   Ni   i    i   s   For server %s, using IP %s(	   t   cfgt   randomize_server_ipR#   t   lent   randomt   randintt   loggingt   debugR   (   R)   t   rndt   ip(    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   hostipX   s    *	c         C   s   x |  j  D] } y | j j j   } Wn d  } n X| rX | | k rX | j |  n  | rz | | k rz | j |  n  | j d t  q
 Wg  |  _  d  S(   Nt   quit(   R   t   nntpt   sockt   filenoR   t   popt	   terminateR   (   R)   t   readerst   writerst   nwt   fno(    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   stopd   s    
c         C   s   d |  j  |  j f S(   Ns   %s:%s(   R   R   (   R)   (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   __repr__q   s    N(	   t   __name__t
   __module__R   R   R+   t   propertyR5   R@   RA   (    (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyR   8   s
   	t
   Downloaderc           B   s  e  Z e Z e d   Z d   Z e d    Z e d    Z	 e e
 d   Z e d    Z e d    Z e d    Z e d    Z d	   Z e d
    Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z e
 e
 e e d  Z d   Z e e  d    Z e e  d    Z e e e  d     Z  d   Z! e e e  d     Z" e d    Z# e d    Z$ d   Z% RS(   c         C   s   t  j |   t j d  | |  _ t j   |  _ t j j |  j  t	 |  _
 t	 |  _ t	 |  _ d |  _ t	 |  _ i  |  _ i  |  _ g  |  _ i  |  _ x$ t j   D] } |  j d  |  q Wt |  j  |  _ |  t _ d  S(   Ns   Initializing downloader/decoderi    (   R    R+   R1   R2   t   pausedR,   t   bandwidth_limitt   callbackt	   speed_setR   t   delayedt   postproct   shutdownt   _Downloader__restartt   force_disconnectt   read_fdst	   write_fdst   serverst   _timerst   configt   get_serverst   init_serverR   R   t   decoderRE   t   do(   R)   RF   t   server(    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyR+   |   s$    										c         C   s  t  } t  } t j   } | | k r | | } | j   } | j   } | j   }	 | j   }
 | j   } | j   } | o | o | d k } | j	   o t
 j j } | j   } | j   } | j   } t | j   d d  } t } n  | rfxo t t |  j   D]U } |  j | j | k r
t  } | |  j | _ t |  j | _ |  j d 7_ Pq
q
Wn  | r| r| r|	 r| r|  j j t | | |	 |
 | | | | | | |   n  | S(   Ni    i   i  i   (   R   RS   RT   t   enableR   R   R   t   connectionsR   R   t   sabnzbdt   newswrappert   HAVE_SSLR   R   R   t   floatR   R   t   xrangeR.   RQ   R   R   R   RM   R(   R   (   R)   t	   oldservert	   newservert   primaryt   createRQ   t   srvt   enabledR   R   R   R   R   R   R   R   R   R   t   n(    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyRU      s<    
	!c         C   s   | |  _  d  S(   N(   RF   (   R)   t   state(    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   set_paused_state   s    c         C   s   t  j d  t |  _ d  S(   Nt   Resuming(   R1   R#   R   RF   (   R)   (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   resume   s    c         C   s   |  j  s t |  _  t j d  t j d t d  d  |  j   rT t j	 j
   n  t j   rm |  j   n  | r t j   q n  d  S(   Nt   Pausingt   SABnzbdt   Pausedt   download(   RF   R   R1   R#   t   growlert   send_notificationt   Tt	   is_pausedR
   RW   t   resetR,   t   autodisconnectt
   disconnectR[   t
   save_state(   R)   t   save(    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   pause   s    		c         C   s   t  j d  t |  _ d  S(   Nt   Delaying(   R1   R2   R   RJ   (   R)   (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   delay   s    c         C   s   t  j d  t |  _ d  S(   Nt
   Undelaying(   R1   R2   R   RJ   (   R)   (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   undelay   s    c         C   s   t  j d  t |  _ d  S(   Ns%   Waiting for post-processing to finish(   R1   R#   R   RK   (   R)   (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   wait_for_postproc   s    c         C   s   t  j d  t |  _ d  S(   Ns+   Post-processing finished, resuming download(   R1   R#   R   RK   (   R)   (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   resume_from_postproc   s    c         C   s   t  |  _ d  S(   N(   R   RN   (   R)   (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyRu      s    c         C   s#   t  |  |  _ t j d |  d  S(   Ns   Bandwidth limit set to %s(   t   intRG   R1   R#   (   R)   t   value(    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   limit_speed   s    c         C   s   |  j  S(   N(   RG   (   R)   (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt	   get_limit  s    c         C   s   t  j   |  _ d  S(   N(   R,   RG   (   R)   (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyRI     s    c         C   s8   d d l  m } |  j s t S| j j   r0 t St Sd  S(   Ni(   t   NzbQueue(   t   sabnzbd.nzbqueueR   RF   R   RW   t   has_forced_itemsR   (   R)   R   (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyRr     s    	c         C   s/   x( |  j  D] } | j r
 | j r
 t Sq
 Wt S(   N(   RQ   R   R   R   R   (   R)   RX   (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   active_primaries  s    c         C   s   t  | j |  j  S(   N(   t   filtert   server_in_try_listRQ   (   R)   t   nzo(    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   nzo_servers  s    c      
   C   s   d d l  m } | j r | j r | j | j d k r d | _ t | _ t d  d t f | _	 t
 j t d  | j t  |  j | j t  x: | j | j D]( } |  j | d d t d	 t d
 t q Wd  | _ | j j   n  d  S(   Ni(   R   i   i    s(   Server %s will be ignored for %s minutesR   s   forcing disconnectt   warnt   waitR6   (   R   R   R   R   R    R   R   Rq   t   _PENALTY_TIMEOUTR!   R1   R"   t   TaR   t   plan_serverR   R   t   _Downloader__reset_nwR   R#   RW   t   reset_all_try_lists(   R)   RX   R   R>   (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   maybe_block_server  s    (		&	c         C   s  d d l  m } |  j j   t j j   xxc|  j D]X} x | j D] } | j	 rc | j	 j
 s | j rH t j   | j k rH | j	 r | j	 j
 r |  j | d d t n |  j | d  | j d 7_ |  j |  qH qH W| j rb| j s7 | j } | j |  j |  j  |  j j |  | r<|  j d  |  n  |  j d 8_ | j j   Pqbq7 n  | j s7 | j s7 |  j   s7 |  j s7 |  j s7 |  j rq7 n  | j  o| j j! |  sq7 n  x| j D]} | j rt j   | j k  rqqd  | _ n  | j  sPn  | j" d  k r;|  j |  t# |  Pn  | j j$ |  } | sWPn  | j% r| j& j' j( t j   | j% k  rt) j* rt+ j, d | j- | j. | j/  n  |  j j0 | d   Pn  | j j |  | j j1 |  | | _- | j2 r|  j3 |  qy3 t+ j" d | j4 | j. | j/  | j5 |  j  Wqt+ j6 t7 d	  | j4 | j. | j/  t+ j" d
 d t8 |  j | d  qXqWq7 W|  j r't8 } x$ |  j D] } | j rt } PqqW| r'|  j j   |  j j9   x' |  j D] } | j |  j |  j  qWt+ j" d  Pq'n  |  j: rxi |  j D]^ } xL | j | j D]: } | j2 of| j  } |  j | d d t d t d | qQWd  | _" q:Wt |  _: n  |  j j;   } |  j j;   }	 | s|	 rt< j< | |	 d9 d  \ }
 } } n g  g  g  }
 } } t j j=   t j> d  t? j@   xO | j jA   s^|  j   s^|  j s^|  j r|  j r|  j rt? jB   q1Wt? jC   t |  _: xn | D]f } |  j | } | j	 jD jE   } | |  j k r| |  j | <n  | |  j k r|  j jF |  qqW|
 st j j   q- n  x|
 D]} |  j | } | j- } | jG } | r^| j& j' } n  y | jH   \ } } } Wn d t t } } } n X| rt j j   q$n  | d k  r|  j | d d t d t q$n |  jI rSt j jJ   } | | 7} |  jI d } | | k rSx6 t j jJ   | k rLt j> d  t j j   qWqSn  t j j | jK |  | r| jL | 7_L | jM t j jJ    n  tN | jO  d k rZ| jO d d  } | j2 s| d k rht } y^ | jP |  t) j* r$t+ j, d | j4 | jG j. | jG j/ | jO d  n  g  | _O d | _Q WntR k
 r} t } d } | jS } | d  } d | } t+ j, d | |  | d: k r(	tT |  r(	| j  rt7 d  d | f | _U t+ j6 t7 d  | j. | j/  |  j | d  d t d t8 d t8 |  jV | jK tW  | jX d 8_X qn| d; k r	tY |  r	| j  rt7 d   | | _U d! | j. | j/ f } t+ j6 t7 d   |  tZ } qn| d< k s	| d k r
t[ |  r
| j  r	t7 d$  | | _U t+ j6 t7 d$  d% | j. | j/ f  n  t\ } t8 } n| d k r
| j  rg
t7 d&  d | f | _U t+ j] t7 d&  d% | j. | j/ f |  n  t^ |  r|
t\ } n t_ } t8 } n | d k r
| j  r
t+ j, d' | j.  n  t` } t8 } nT | j  rt7 d&  d | f | _U t+ j6 t7 d&  d% | j. | j/ f |  ta } n  | s.| r$| jb r$| j  rt | _  | jb r_tc jd   r_te } n  | r| st| jb rt+ j" d( | jK |  |  jV | jK |  n  | j j   n  |  j | d  d t d t8 q$q$nR t+ j6 t7 d)  | j4 | jG j. | jG j/ | jO d  |  j | d  d t n X| j2 rWt+ j" d* | j4 | jG j. | jG j/  |  j3 |  qWqZ| d+ k rt8 } t+ j, d, | j-  |  j j0 | | jO  qZ| d- k rt } t+ j, d. | jO  | j- j& j' jf | _f g  | _O d | _Q |  j3 |  qZ| d= k rRt8 } d  | _O t+ j" d2 d3 | j4 | jG j. | jG j/ | j- |  qZ| d k r| j  rt | _  tg d4  d | _U |  jV | jK d  | j j   n  tg d4  d% | jG j. | jG j/ f } |  j | | d t8 qZ| d5 k rZ| jh rt | _i t+ j, d6 | j.  n t | _j t+ j, d7 | j.  g  | _O d | _Q |  j3 |  qZn  | r$d | _ t) j* rt+ j, d8 | j4 | j. | j/ | j-  n  |  j j0 | | jO  | jk   | j j |  | j j1 |  q$q$Wq- Wd  S(>   Ni(   R   R   R   s	   timed outi   s   Article %s too old for %s:%ss   %s@%s:%s: Initiating connections   Failed to initialize %s@%s:%ss   Traceback: t   exc_infos   failed to initializes   Shutting downs   forcing disconnectR   R6   g      ?i    s   server closed connectioni   g?i   t   480s   %s@%s:%s last message -> %ss    [%s]s   Server login problem: %s, %st   502t   481t   400s$   Too many connections to server %s:%st   destroys   Probable account sharings    (%s:%s)t   482t   381s   Failed login for server %ss   %s:%ss    Cannot connect to server %s [%s]s$   Unspecified error 400 from server %ss    Server %s ignored for %s minutess&   Connecting %s@%s:%s failed, message=%ss   Connecting %s@%s:%s finishedt   223s   Article <%s> is presentt   211s   group command ok -> %st   411t   423t   430s   Thread %s@%s:%s: Article s   %s missing (error=%s)s    Server %s requires user/passwordt   500s   Server %s does not support STATs   Server %s does not support BODYs   Thread %s@%s:%s: %s done(    (   R   R   R   (   R   R   (   R   R   R   (   R   R   R   (l   R   R   RV   t   startR
   RW   t   updateRQ   R   R7   t	   error_msgR   t   timeR   R   R    R   R   R   R@   RO   RP   t   removeRU   R   RM   R   R   Rr   RL   RJ   RK   R   t   has_articles_forR#   R   t   get_articleR   t   nzfR   t	   avg_stampR[   t   LOG_ALLR1   R2   t   articleR   R   t   decodeR(   t	   connectedt   _Downloader__request_articlet   thrdnumt   init_connectt   errorR   R   t   joinRN   t   keyst   selectRs   t   sleepR   t   acquiret   is_emptyR   t   releaseR8   R9   R:   RX   t
   recv_chunkRG   t   get_bpsR   t   bytes_downloadedt   update_avg_kbsR.   t   linest   finish_connectt   dataR   t   responset   clues_too_manyR!   R   t   _PENALTY_TOOMANYR   t   clues_too_many_ipt   _PENALTY_SHAREt   clues_logint   _PENALTY_PERMR"   t	   clues_payt   _PENALTY_502t   _PENALTY_VERYSHORTt   _PENALTY_UNKNOWNR   R,   t   no_penaltiest   _PENALTY_SHORTt   groupRq   t   precheckR&   R%   t
   soft_reset(   R)   R   RX   R>   R   R   t   emptyR6   t   readkeyst	   writekeyst   readt   writeR   t   selectedR9   R   t   bytest   donet   skipt   bpst   limitt   codet   blockt   penaltyt   msgt   ecodet   display_msgt   name(    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   run/  s   3			:		
+	"			
			&$
0
				
			

	"	$	)		,					)				 			
						&							
c         C   sZ   x( |  j  D] } |  j  | | k r
 | Sq
 Wx( |  j D] } |  j  | | k r5 | Sq5 Wd  S(   N(   RO   RP   R   (   R)   R>   t   f(    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   __lookup_nw}  s    c         C   s!  d d l  m } | j } | j }	 d  }
 | j ru y | j j j   }
 Wn |  j |  }
 t	 } n Xd  | j _
 n  | r | r | | _ t j d | | j | j | j  n, | r t j d | | j | j | j  n  | | j k r | j j |  n  | p| | j k s&| j j |  n  |
 rN|
 |  j k rN|  j j |
  n  |
 rv|
 |  j k rv|  j j |
  n  |	 r|	 j t j   k r|	 j j st j   r|  j j |	 d   qd  |	 _ |	 j  } | j! } | j" j# | |  n  | r
| j$ d |  n | j% | d | d  S(   Ni(   R   s   Thread %s@%s:%s: R6   (&   R   R   RX   R   R   R7   R8   R9   t   _Downloader__lookup_nwR   R   R"   R1   R#   R   R   R   R   R   R   R(   RP   R:   RO   t   triesR,   t   max_art_triest   fetcherR   t   max_art_optRV   R   R   R   RW   t   reset_try_listsR;   t
   hard_reset(   R)   R>   R!   R   R   R   R6   R   RX   R   R9   R   R   (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt
   __reset_nw  sH    			
	.			c         C   sx  y | j  j j } t j   r} | j | j k r} | j } t j rm t j	 d | j
 | j j | j j |  n  | j |  nJ t j r t j	 d | j
 | j j | j j | j  j   n  | j | j  | j j j   } | |  j k r | |  j | <n  Wnx t j k
 r7} t j d |  |  j | d d t n= t j d  t j d d t |  j | d d t n Xd  S(	   Ns   Thread %s@%s:%s: GROUP <%s>s   Thread %s@%s:%s: BODY %ss'   Looks like server closed connection: %ss   server broke off connectionR6   s   Suspect error in downloaders   Traceback: R   (   R   R   R   R,   t
   send_groupR   R[   R   R1   R2   R   RX   R   R   t   bodyR   R7   R8   R9   RO   t   socketR   R#   R   R   R   (   R)   R>   R   R   R9   t   err(    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   __request_article  s,    			c         C   s   t  j d | |  | |  j k r2 g  |  j | <n  t j   d | } |  j | j |  | r t j j |  j | | g |  n  d  S(   Ns'   Set planned server resume %s in %s minsg      N@(	   R1   R2   RR   R   R(   R[   t	   schedulerR   t   trigger_server(   R)   t	   server_idt   intervalt   stamp(    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyR     s    c         C   sV   t  j d |  | |  j k rR | |  j | k rR |  j | =|  j | |  qR n  d  S(   Ns    Trigger planned server resume %s(   R1   R2   RR   RU   (   R)   R   t	   timestamp(    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyR     s
    
c         C   sw   y |  j  | =Wn t k
 r! n XxN |  j D]C } | j | k r, | j r, t j d |  |  j | |  Pq, q, Wd  S(   Ns   Unblock server %s(   RR   t   KeyErrorRQ   R   R   R1   R2   RU   (   R)   R   RX   (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   unblock  s    c         C   s+   x$ |  j  j   D] } |  j |  q Wd  S(   N(   RR   R   R   (   R)   R   (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   unblock_all  s    c         C   s  t  j    } g  } x} |  j j   D]l } g  |  j | D] } | | k r6 | ^ q6 s" t j d |  |  j | =|  j | |  | j |  q" q" Wxk |  j D]` } | j |  j k r | j | k r | j	 r t j d | j  |  j | j | j  q q q Wd  S(   Ns"   Forcing re-evaluation of server %ss   Forcing activation of server %s(
   R   RR   R   R1   R2   RU   R(   RQ   R   R   (   R)   t   nowt   kickedR   R   RX   (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   check_timers  s    ,
c         C   s   |  j  | |  d  S(   N(   RU   (   R)   R`   Ra   (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   update_server  s    c         C   s   d  S(   N(    (   R)   (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   wakeup  s    c         C   s&   t  |  _ t j d t d  d  d  S(   NRl   s   Shutting downt   startup(   R   RL   Ro   Rp   Rq   (   R)   (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyR@   !  s    	(&   RB   RC   R   RW   R   R+   RU   R   Rh   Rj   R   Rx   Rz   R|   R}   R~   Ru   R   R   RI   Rr   R   R   R   R   R   R   R   R   t
   TIMER_LOCKR   R   R   R   R   R   R   R@   (    (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyRE   w   sD   %	+				
				 O	
3	!		c           C   sR   t  j   z t j j   Wd  t  j   t  j   Xy t j j   Wn n Xd  S(   N(   R   R   RE   RW   R@   t	   notifyAllR   R   (    (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyR@   &  s    

c         C   s1   |  j    }  x d D] } | |  k r t Sq Wt S(   NR   R   t   invalidt   authens   access denied(   R   R   R  R  s   access denied(   t   lowerR   R   (   t   textt   clue(    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyR   4  s
    c         C   s1   |  j    }  x d D] } | |  k r t Sq Wt S(   Nt   exceedRZ   s   too manyR   R   (   R  RZ   s   too manyR   R   (   R  R   R   (   R  R  (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyR   >  s
    c         C   s1   |  j    }  x d D] } | |  k r t Sq Wt S(   Ns   simultaneous ips   multiple ip(   s   simultaneous ips   multiple ip(   R  R   R   (   R  R  (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyR   H  s
    c         C   s1   |  j    }  x d D] } | |  k r t Sq Wt S(   Nt   creditst   paymt   expired(   R	  R
  R  (   R  R   R   (   R  R  (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyR   R  s
    (/   R   R   R1   t	   threadingR    R   t   nntplibR   R   R/   R[   t   sabnzbd.decoratorsR   R   R   t   sabnzbd.decoderR   t   sabnzbd.newswrapperR   R   t   sabnzbd.growlerRo   t   sabnzbd.constantst   sabnzbd.configRS   t   sabnzbd.cfgR,   t   sabnzbd.bpsmeterR
   t   sabnzbd.schedulerR   R   R   R   R   R   R   R   R  t   objectR   RE   R@   R   R   R   R   (    (    (    s&   /var/www/SABnzbd/sabnzbd/downloader.pyt   <module>   sD   
	?  		
	
	
