From efb85a3634f6e8445abf7b9ec9c069c78ec43fcb Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Mon, 10 Mar 2025 11:19:27 +0200 Subject: [PATCH] working training on CPU --- crypto/gogo2/main.py | 131 +- ...s.out.tfevents.1741596450.GW-DOBRI.46872.0 | Bin 0 -> 88 bytes ...s.out.tfevents.1741596799.GW-DOBRI.38248.0 | Bin 0 -> 88 bytes ...s.out.tfevents.1741598305.GW-DOBRI.61848.0 | Bin 0 -> 52649 bytes crypto/gogo2/trading_bot.log | 8325 +++++++++++++++++ 5 files changed, 8415 insertions(+), 41 deletions(-) create mode 100644 crypto/gogo2/runs/trading_agent/events.out.tfevents.1741596450.GW-DOBRI.46872.0 create mode 100644 crypto/gogo2/runs/trading_agent/events.out.tfevents.1741596799.GW-DOBRI.38248.0 create mode 100644 crypto/gogo2/runs/trading_agent/events.out.tfevents.1741598305.GW-DOBRI.61848.0 diff --git a/crypto/gogo2/main.py b/crypto/gogo2/main.py index 2e483c0..05d1e82 100644 --- a/crypto/gogo2/main.py +++ b/crypto/gogo2/main.py @@ -17,6 +17,7 @@ from dotenv import load_dotenv import ccxt import websockets from torch.utils.tensorboard import SummaryWriter +import torch.cuda.amp as amp # Add this import at the top # Configure logging logging.basicConfig( @@ -63,7 +64,7 @@ class ReplayMemory: return len(self.memory) class DQN(nn.Module): - def __init__(self, state_size, action_size, hidden_size=256, lstm_layers=2, attention_heads=4): + def __init__(self, state_size, action_size, hidden_size=384, lstm_layers=2, attention_heads=4): super(DQN, self).__init__() self.state_size = state_size @@ -73,9 +74,10 @@ class DQN(nn.Module): # Initial feature extraction self.fc1 = nn.Linear(state_size, hidden_size) self.bn1 = nn.BatchNorm1d(hidden_size) + self.dropout1 = nn.Dropout(0.2) # Add dropout for regularization # LSTM layer for sequential data - self.lstm = nn.LSTM(hidden_size, hidden_size, num_layers=lstm_layers, batch_first=True) + self.lstm = nn.LSTM(hidden_size, hidden_size, num_layers=lstm_layers, batch_first=True, dropout=0.2) # Attention mechanism self.attention = nn.MultiheadAttention(hidden_size, attention_heads) @@ -83,6 +85,7 @@ class DQN(nn.Module): # Output layers with increased capacity self.fc2 = nn.Linear(hidden_size, hidden_size) self.bn2 = nn.BatchNorm1d(hidden_size) + self.dropout2 = nn.Dropout(0.2) self.fc3 = nn.Linear(hidden_size, hidden_size // 2) # Dueling DQN architecture @@ -90,7 +93,7 @@ class DQN(nn.Module): self.advantage_stream = nn.Linear(hidden_size // 2, action_size) # Transformer encoder for more complex pattern recognition - encoder_layer = nn.TransformerEncoderLayer(d_model=hidden_size, nhead=attention_heads) + encoder_layer = nn.TransformerEncoderLayer(d_model=hidden_size, nhead=attention_heads, dropout=0.1) self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=2) def forward(self, x): @@ -105,16 +108,15 @@ class DQN(nn.Module): # Handle mismatched input by either truncating or padding if x.size(1) > self.state_size: x = x[:, :self.state_size] # Truncate - print(f"Warning: Input truncated from {x.size(1)} to {self.state_size}") else: # Pad with zeros padding = torch.zeros(batch_size, self.state_size - x.size(1), device=x.device) x = torch.cat([x, padding], dim=1) - print(f"Warning: Input padded from {x.size(1) - padding.size(1)} to {self.state_size}") # Initial feature extraction x = self.fc1(x) x = F.relu(self.bn1(x) if batch_size > 1 else self.bn1(x.unsqueeze(0)).squeeze(0)) + x = self.dropout1(x) # Reshape for LSTM x_lstm = x.unsqueeze(1) if x.dim() == 2 else x @@ -134,6 +136,7 @@ class DQN(nn.Module): # Final layers x = self.fc2(x) x = F.relu(self.bn2(x) if batch_size > 1 else self.bn2(x.unsqueeze(0)).squeeze(0)) + x = self.dropout2(x) x = F.relu(self.fc3(x)) # Dueling architecture @@ -641,6 +644,12 @@ class Agent: self.device = device self.memory = ReplayMemory(MEMORY_SIZE) + # Configure for RTX 4060 (8GB VRAM) + if device == "cuda": + torch.backends.cudnn.benchmark = True # Optimize for fixed input sizes + logger.info(f"Using GPU: {torch.cuda.get_device_name(0)}") + logger.info(f"Available VRAM: {torch.cuda.get_device_properties(0).total_memory / 1e9:.2f} GB") + # Q-Networks with configurable size self.policy_net = DQN(state_size, action_size, hidden_size, lstm_layers, attention_heads).to(device) self.target_net = DQN(state_size, action_size, hidden_size, lstm_layers, attention_heads).to(device) @@ -653,12 +662,19 @@ class Agent: self.optimizer = optim.Adam(self.policy_net.parameters(), lr=LEARNING_RATE) + # Mixed precision training + self.scaler = amp.GradScaler() + self.use_amp = device == "cuda" # Only use mixed precision on GPU + self.epsilon = EPSILON_START self.steps_done = 0 # TensorBoard logging self.writer = SummaryWriter(log_dir='runs/trading_agent') + # Create models directory if it doesn't exist + os.makedirs("models", exist_ok=True) + def expand_model(self, new_state_size, new_hidden_size=512, new_lstm_layers=3, new_attention_heads=8): """Expand the model to handle more features or increase capacity""" logger.info(f"Expanding model: {self.state_size} → {new_state_size}, " @@ -726,46 +742,79 @@ class Agent: return random.randrange(self.action_size) def learn(self): + """Learn from experience replay with mixed precision""" if len(self.memory) < BATCH_SIZE: return None - experiences = self.memory.sample(BATCH_SIZE) - batch = Experience(*zip(*experiences)) - - # Convert to tensors - state_batch = torch.FloatTensor(batch.state).to(self.device) - action_batch = torch.LongTensor(batch.action).unsqueeze(1).to(self.device) - reward_batch = torch.FloatTensor(batch.reward).to(self.device) - next_state_batch = torch.FloatTensor(batch.next_state).to(self.device) - done_batch = torch.FloatTensor(batch.done).to(self.device) - - # Get Q values for chosen actions - q_values = self.policy_net(state_batch).gather(1, action_batch) - - # Double DQN: use policy net to select actions, target net to evaluate - with torch.no_grad(): - # Get actions from policy net - next_actions = self.policy_net(next_state_batch).max(1)[1].unsqueeze(1) - # Evaluate using target net - next_q_values = self.target_net(next_state_batch).gather(1, next_actions) - next_q_values = next_q_values.squeeze(1) + try: + # Sample batch from memory + experiences = self.memory.sample(BATCH_SIZE) - # Compute target Q values - expected_q_values = reward_batch + (GAMMA * next_q_values * (1 - done_batch)) - expected_q_values = expected_q_values.unsqueeze(1) - - # Compute loss (Huber loss for stability) - loss = F.smooth_l1_loss(q_values, expected_q_values) - - # Optimize the model - self.optimizer.zero_grad() - loss.backward() - # Gradient clipping - for param in self.policy_net.parameters(): - param.grad.data.clamp_(-1, 1) - self.optimizer.step() - - return loss.item() + # Check if any experience has None values + for exp in experiences: + if exp.state is None or exp.next_state is None: + return None + + # Convert to tensors + states = torch.FloatTensor([exp.state for exp in experiences]).to(self.device) + actions = torch.LongTensor([exp.action for exp in experiences]).unsqueeze(1).to(self.device) + rewards = torch.FloatTensor([exp.reward for exp in experiences]).to(self.device) + next_states = torch.FloatTensor([exp.next_state for exp in experiences]).to(self.device) + dones = torch.FloatTensor([exp.done for exp in experiences]).to(self.device) + + # Use mixed precision for forward/backward passes + if self.use_amp: + with amp.autocast(): + # Compute Q values + current_q_values = self.policy_net(states).gather(1, actions) + + # Compute next state values using target network + with torch.no_grad(): + next_q_values = self.target_net(next_states).max(1)[0] + target_q_values = rewards + (GAMMA * next_q_values * (1 - dones)) + + # Reshape target values to match current_q_values + target_q_values = target_q_values.unsqueeze(1) + + # Compute loss + loss = F.smooth_l1_loss(current_q_values, target_q_values) + + # Optimize with gradient scaling + self.optimizer.zero_grad() + self.scaler.scale(loss).backward() + self.scaler.unscale_(self.optimizer) + torch.nn.utils.clip_grad_norm_(self.policy_net.parameters(), 1.0) + self.scaler.step(self.optimizer) + self.scaler.update() + else: + # Standard precision training + # Compute Q values + current_q_values = self.policy_net(states).gather(1, actions) + + # Compute next state values using target network + with torch.no_grad(): + next_q_values = self.target_net(next_states).max(1)[0] + target_q_values = rewards + (GAMMA * next_q_values * (1 - dones)) + + # Reshape target values to match current_q_values + target_q_values = target_q_values.unsqueeze(1) + + # Compute loss + loss = F.smooth_l1_loss(current_q_values, target_q_values) + + # Optimize the model + self.optimizer.zero_grad() + loss.backward() + torch.nn.utils.clip_grad_norm_(self.policy_net.parameters(), 1.0) + self.optimizer.step() + + return loss.item() + + except Exception as e: + logger.error(f"Error during learning: {e}") + import traceback + logger.error(traceback.format_exc()) + return None def update_target_network(self): self.target_net.load_state_dict(self.policy_net.state_dict()) diff --git a/crypto/gogo2/runs/trading_agent/events.out.tfevents.1741596450.GW-DOBRI.46872.0 b/crypto/gogo2/runs/trading_agent/events.out.tfevents.1741596450.GW-DOBRI.46872.0 new file mode 100644 index 0000000000000000000000000000000000000000..8a2611d612373a973b619393e849c750bb41bd34 GIT binary patch literal 88 zcmeZZfPjCKJmzx#7VSK-^7BncDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 h(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNXP@j4ghRrAw>WH literal 0 HcmV?d00001 diff --git a/crypto/gogo2/runs/trading_agent/events.out.tfevents.1741596799.GW-DOBRI.38248.0 b/crypto/gogo2/runs/trading_agent/events.out.tfevents.1741596799.GW-DOBRI.38248.0 new file mode 100644 index 0000000000000000000000000000000000000000..da2acc3dd4714b025d1fb83f7ef57b4ea9468c30 GIT binary patch literal 88 zcmeZZfPjCKJmzvf+4@X=)#sazQoKn;iJ5tNu4SotC00g3dR#gssd>fuMM?RIMJam4 hrMbC@MU{HxMVTe3MS7_qRq<(=IjQjwk@wx9g#ducA=Ur@ literal 0 HcmV?d00001 diff --git a/crypto/gogo2/runs/trading_agent/events.out.tfevents.1741598305.GW-DOBRI.61848.0 b/crypto/gogo2/runs/trading_agent/events.out.tfevents.1741598305.GW-DOBRI.61848.0 new file mode 100644 index 0000000000000000000000000000000000000000..75c51bf7dfac42e239e341dc0070ea88c271552d GIT binary patch literal 52649 zcmaLgcU;fy`#11}Bq1vq*()PN%6WcBAuA*)k`hH$NQ8{G(k@L6X=rGwl+q-Xk+Nwc zib4y``}$qS@4g?8<9mD`w?D7P>)COhpU-=o=kb2mwO$DR^?I%k8mtz3&@Z<2o$~mh z`%DcEnaHj^V`O5+eNEB|0yxXU3B`20uP(_ zDBp7WGUSy>zCxi#&@A2a8s*jBSVLZgMId+g=P4aast_vJ5?9@Yf zmHm#8UqkXIvdRS#AKy$w`JX1vkY7vkOP)Ily1&%~qP)rxcgU-feBW^g!I)%=LnuFW zw-@Bsk^G@wo`SG*D%~ir`Q8We>q)-kVVK~hVpIdle;5@2`3)qW>SQbMvr|h&`R6e~ zkl#r1>mO_pSSl{{NBQghLLtA2zE9 zW|FV-SSm1>s`nPFOR`RtKdkl#-7 zUyHQ`p_B5HQC_Y+2lAREpFU!N;E0^VIh3CuD1iJ9lDBxQC@4r=Sc>w7riGB#BKeUg z#01BewoO9$?Iy*L*CzQx1&ak$Pp5rA`9Sd}kl#u2vPyD-$8L@IdKDa53i(|mKR|hk zVCMu4E%fux+$x8>4#|&oA1m0pUdjOF9YkJ3emBWa+^Z}w)zJ1u`Sy=*A+JmFmc3Gf z0x`K-l&>%N0Qo&6uW`PI|Cm>4j`EwvRzqHoLY?*tJ&-^FJ|nFQ@&`#?=ivas_m+rUlt0VUpi2TF=YoZ+1cXzMZ0wKSJ`EM?(4XbCNbF z|1@+6$Fu<-atKj6r$jjmsc^j^yo(n)$M3J`?5rOqWCcJjuT<`NLB`%i`i}hk2~^=r>wS_hVo;4*Fydh$$va5 zDiEDBLV)rUb=E`vGRf~YEaGihIXzQ_;9ozB?;V3D+i!xrWj_KoyuHn{Hh7*u;Ary} z09F)8+SqZohpQhU&?%$=z?uR^Zo3Iv01^4Y_ptq`z@IS9a!0@WygelvOL4g}I49tLogg8nl?d1v*HbqM^v ze++;V1?O(oa`okV4kKXo^8^5A3LbP`;x3QRmmwh6ehPpK1pxt%`6$UITLh*YHV5EJ z!C*rh{>s!K8G#t5^8nl^IHan^Kf3-ng1~6givZjyunf=Sl5E}$1T5k$0eDa#u62!D z`xn(CAl7aJz>@;G^UwG|i`;z(M327$z>9+WGoAUdJg01Q4VdQyVD1>PQDWj6M&|!? z2L5+XAK@%f2uOtHv#xia4Wt* z(9QyPqY8^dA^`YOFt{U0uzOwEN(A`*C;)yG2+Mg0daq3_#^6^h0DlTL>IMo*mOKU_KxfKp+L8O*;g9@aRwkYE9Dt z1W_{Aee&mrbU8dmk)a*5H}?oKnMkMqL^U*+O)w4dU+5gjJ5kx?3gKy7IWfG`TaPM9kw zO4pi+fUUz*0O1tukyI2s{ic$NKz8MG01*^K*3J^B`b9V(VEOX}fJh2XIJWYfJ8uUd z;BE8(a!6Qky8oCu1d8e_0YpLibfLIEAg|F~igS+tgkX}#^;5G$?{W7^n)a{?B!jO~(0C5y_ z+YAtRUG>M8nc1pM0OBdw5L300OV2Nz21|rRbTTARoJ?CA%J`eHnd;kU3(kw9jHfB4nP3~NAGJ1B#y6=L=_BD zeVX;=nV>9)a}tlV#^07@v>_sNl~ z_dJt9LmF^sBY?*gG}S)jlTU`=6UgKFW&lqpSdhG#+Yee+gDOOIs{?pS!ETWdzHa{G z;|RQK-VWdy1*a75a?wHezagN%RvSPm1rfhSQY|`=M4e8prD`56|UO8F9(64vc>>j zQeb8m%3q~rrXz4@Yx{xz~Hs6a=yf7e-GeA431^Jqu|m7@VPP%t#tg^#Kcj6`6R!fgPJ6ezdr@)$kyb_5J2 zCjj_Hf$sj(eEL!`yhzmDlMLWH1rK*c@IO_HaJMMQB^5vu1(G!f`SqqJI%r7OGSUGw zQ!szPWiGlr?GgfAFEauBpkR9A2EO-A2OiS9%{c&mQc(A?mRI;3#pgqQUmk#86bu-A ziA&i`9fXFY82u1H3k9L+$GBn8`0)scTNVRor9k7z0lt4^=Mn_ISUmyIMnS*QNS@<& zSO|d;OG^Q?Q($tzn8zIOTZ4dzc{zX%3T7O);$^$O$05)d^a{Xl3exgqc-q!8lMv{s zdkf$X1*tW@T)l8K-p3hAQ~>x(fzE;m9zSYbKUBfZtQtTk1qU=$_~iA*_|Q1Hs}4XH z1;3x&;6rmJ;vSi|>SqAm6rArb%a?rLh_}b%g1!Rip+Ms4dY%}aiLc%hwtfT9OTpCP znmk6~_eC^msiI~8eH5I1qQNbt9GekvP5A}DQkXmuDSd3m7c9uQj(~1=8vttxde^G) z>XYVp3HN*89{@HKSZ=f7xubneQH6(_x&YWx@Kk6bFVi{55fI+g3&4(oPY?Wfhvmoz z2rSd@cm3ZDxIG1<8;tnbkV;kJOO+xwU2T_(D%{>A55Sj#W^orDvf3W+MMDoO0`Q|?Sf?TX)L6I$RcL<4 z0Qgfd_u>-X(KreB?%ga_0tleswTU?oGP<%KRoJ#t8NhW4hQB|@qdcWPBcQTt4S+xj z1fzYq*rtau2CQ45kN2nYgWzW%ENC3qY4vmZw3%TK~>f| z{vyE#-xQ5Q)dAd~Am!8^?tI<`FMq}+YzGiZL515QZZta#w~I~$YXP`P!KZPn__8t6 z@!I>D^)3Km6g(e$n3uZ<6wst+`{@D*r@-BH9+z+w2qSR4S|30J1x;`4_?@h@-v~6% z-VY#>f{9$2uP9zH5P?pW0|0JOQ0ldjYxLXk5`nAR4grXw;P04Kd`;_U-0TZAxsT8)J5<5z+Hn9e6f|Y*<>B!I@T3d3m;#8UU}}gQcUq`E8C976`ZR#s6pZIv zd0&Z04Fa3O&H{*|z;(`LUe_eI7XkS~3jpyH{Ed<0^9t_tM?mAvB>)K&l)I1Q!Yh@W z5ST7x4Iq(%Iw4bj^LbAK0=(K5KoSMc;s?3V&O7H2__W0VKr#gf_Dti#eWmz(xV_d1 zz#R&1m+J6a5*c`>Xtl)^Knevr63sq*Gw(b(+8N{Zff4_*w@7$eaRhLiC)R#vUwkak9 z$fe-A{6g+f`6~&5)D5Ws@+c^B9>;?tnyV1F5}Xbop8_Q*2fn?uc`X7P-7^6cP*A?? z3csdv2XARsXl4U=K!LCBey&x(#88FqgggKbDHz~v!Z)_Msvr;_^#DL21wE(sa<3~s zc&%~x{UZR6D7ds^4ljA>Fa%Y&-CP2oh=TAwJATMks2qW>FP;J@rodj}2v?exg;xT~ zea``uP~bS_0Dp6m;f}7d+Y11XDbSfRkgwg>i&tdnMsEN-q2Rg<=K?MHXf%gOHSYjC zrQrCErTn`3nym;7n_UUu83jkz8gr9@l_L-cEvN=iNZk+ooB}DK zsl2v0OAl4(=lU5y83ob9NAaJox9}aPb5H|-atgY=6?t6iWc)gx&fxC=UQqD%>qMTq z{ii(|QpW5b0A5m1HCT^(Ex*19fyM)QR0DPd}*wxK^gz{s2@V(nI7(fLD>Gg_Sbi!Hu8qCRU z;s7cs*da5JADY==jD|F2{%`=*6vW(A;;*h5|3M(PY$Sjh3Jew*@Fn(%__murOcFpX z1tt;G_=0{FPf&%194P>G6wF?5hPUYd$U{KCWCDPX6wEby$FlST)DSozIvKzx3f5RJ z;meQ8e?UO|%TxfLDKK1dllRp4Za|<_bS8j$3ZC8^#mjnEFa#o}&H?a+f;nHsxZCA8 zTLkh>&I9n3f{Udh{ME*aiwG2Mmj%#3LAZ|wpPW{PdjdAfivcuJU~)@}2m43h!~1dl z5&+*Q_++ESrN=mJM?;!2Q~|(u3hpG&FvaEz6R9r18;)CMD*!Z8 zknu^LR}PtrpIW|+Sq0z+1@RUGxk}OBlW0iU%4-1pq+r|08NB*&Ul9UphOPtfi-Plo zi@02yRu=+&`WpeXP#|M9i~p@p$L}R4du;~LN&bzbTNqxrEnDU5=Nb z;R(6`{!nnSb2=Zet`;xWU-;?+_)Edz6vhwT9eD{2DaLajfKCd2jc8yur>?=fEH_^x z09_O~SFPdV{gtv&h3&_T0d!MfI^LJvov>~X0*!%30Q69>v{Q{wOz(6@pv24sKraOe zC)=6X4@3O)cyjPb0DTlZzc`wQdGc(^$LD+nuRv;y^0^wiNVvPvh3Y zUHHX?q*lhVu!z5;mxUPP07#dkS<^#&N%Wwlfh}J^dY0q4+@I@tl%w+D*O>xUla|%lY+Ko-&uF4 zbP57l(s2O1D5%-9jtdv>z{gUeRw96F6nI~p#;w|HmZ1v0$L;{|rl3NlAO9luZYlz& z0@DEaP$0B+89)0cVKoBNEHeQ3QXqU$h=2Wd9)Az*_E`Y@C>Zt692Pt0c%up?FLMC+ zQ{a}ri))FQ_98H6O96lY3Vs|t%a@EU#Y+b=xFyWn%c?%FYkX{NPn1W`1Szh)vI|6}Q@5=#%P~bNG2lENO zr zDbP-r;*K2$N)QO^_YJ@;3gpG7@mU7d^$1jqY6cKRfo7&Ych_wQN8qr_PXN&rj9R;a z2g}6Z`}D_QZ2)2@7;90&zW<6TL=|qR{00z9LDj%Pe0<`#UkEH3+X>({1ufxHJbq*N zK?F2)dH}>xpm%aMj}bj)ih$u>p`d?v>b}+@pmx4LfJ6#zu2{|$%ewKE zk7;YV%E@>93u&+fFKr#iJ`$TwC>YFJDJaQBVaEAh;*==m(%(PSlW_S+= zkU~N7khxrEWh=hz)|ih3kV?VUs>__+H^mR|Iz7h#NTWcyRf1Q!bPYs9nvx_1;4TI6 zTc5L4zfL(I;NLs}Ksp84p~Lxn^(Z`~{cV!~WKf{8Pndu2)4GEyO#V9+zsXN=?V zmlg#h5ItljfGi5C4*z8a_x9s^+%Ul$0QV_ankK?U);L9>3T`^{0Ay2O6EmDYa(jx` z-d6Kv0pw6Hr12gL8uJqOj3>J<29QfZ>OKK?-smZchGg=3DS$i*;sV6EM22(0TfVRQeDMDuS?^`;9#~Azyk`jG#SqyJx&5uP!UlE@Q{Kx z{?C~Cqji-CA2ppb$EyCnHq=>~k&nEGB7z#|GSMr!bmTiHud1#gQD0E#GBwKSB? zwUPOZK+bM80L2vOSUzACGWL-OOkKVeKnVqFJ{0iXLnq8LgxnDsN&7nSIH-Kjpgm^q<<43jN4S4oN zJpiQ?G>80T+70KWP=z%+3;;Z*;BLkzc5m-<5d8xUv`90c%^ zg1@)^Fr%Eg7Z6x};4pw!6ioO%h`;0QXAzj|cMQO53MNS>u#&oB{KPLg{RDtF6r?@v zW`@iB@UyH1>8Aj^r9l4gI9|UlU>F+GRv~i$?N^MEJq0SeJ(*!s zGhXuL|F;P{5LYO5-ifp*fxaK2o5wd>WtlDj0XL2mp-~7$gtidrMR1Be2vY3cxoC&P?jht+UqcLqJA07QlB349+}ZYyGDB zB4DHu51@&H1T{@A^yQ&D0^<`C0W?#vXMZ#6Sn<0W0gs>*06!?$wDtqrTQCIQ+EXA%J!Y7Dwi>O)H9zp$Z?56$9v?V9l(j zEIlyT4}r4SCjfp^P`fviN&1L+A~0%kDS$r|yb4TXCC3l;LqJ5S9Kc@+Du=A$3aaN7 z5XcAg*?adpsc1eVuV z0_dS2zr2zuXMUK0fYFIs0KF89jcH_tlK=6>pt{{B0DTl}`z*?}_MFArfj60709Xzn zjq+($tt?dH3*LYSRyP8$rodNGf*XyKR7G?6I=&fz4F%(sUoesVpT8oIuJ#LnEd_Re zXYt4To!k&G{n84+jsl64Y5Z}?7d)ir)*S%sDbVn~&3Z&G;k)Re$iD!tP;jwqGB){C)Rv*R8|w$Lbk31b_<#u4~(vWNk6-{>X(7 z1K>)5+|1kT_cU4Dg*)=2!q8 z6vS?;XMVE5cmtkWF&=;?1v0Lp-0!D_A{tWJ=t%&)C^)jDmAOa$!j~D9*HZvoqhOrH zYj)wx4SYV#l9~a)n}XcHe;(B;My8=59UUkTAp1)OfG-76L(^E_ z%Qc@62xyxRz>fl@2RW>;w6 z+=G_^2&7>B=}KlF=sOcvIJg`@5Cv+j&zOF|9y|wjS^yxJ0w4KKR$ZyN6IIw2p#&g= zf=6TDv*Qgq_?hAgGZg@#6ezqO&Y$=Hg;xSy!D|8Bq(E`xYj$gnvo0Ev;lA|%!YJ4s z)Xe55$m5=|!?{fW!YO#B{D3*T)-OU8LTt7Gh@il7?ig-Sz9$objkmV}h@@b|)(0$Q z{k_)+6wJ~DaEpQ#X(gVMQ;xUC-ydoLh@v32vp-j!A!LRsc&^d`5KY1F`<#!t=YTg& zCi%JmVki*pAI`d;sN ziFzR}H^m3lRh%g0ze7{Vdd|bY`U@o0-qmU1dvKW|1AM*-&BUT$NQdG0=P>-OJz1|9=vfG zs?aQN10bD(W}W_ghiqjl0`W2S0Pa!Hl;+CrSafG0@XOEu{B#S?#H=Er6+^4`IB!kU;dk$|0_M~|N$fn@9%6(R}@@ftm(%l2z0CFg> z`TdUNRs`WKet@$dfLsc!eIi+XU6B~7@cq$s0C^N_P#wu@ZNhL*pdd0BKt2V>q-&Y( zW`o73!m!t&017BD(XM8_TgQ1L&?giD;K6?ZUvt?x%RCzdYNka2ct}CTg?3gYUyGO0 zwo_sO6jG2P*T^LL=NY34MOASC9#PP)p3U~Zh#rT)?EOgqiYTZujAe&&lkmEA`nePU z#T3Z44C1oZn--u70rGbNlu)qd$PnHl{zwaftjco)k7uT0aeeH&*unJfUFg z`a-r%UVuN=;F??jPbp~o^pK6%zE%fSSR(fTz%vToJNU4v<>F%z@RE51pp=3p?JibO zIQ=F9PRbSnyYXVo78H-;bnY`jDfHDfw?x!;&`BC@@`0bU?0hCiPzOI@n^_k&z z(Y0MK0KA~UbaNjo79L@X=CC^GHGr2C)KAW5RuZjv`I8>~4!|o4n&&@ezmHABZM#(k z6#!mSFx)MR9sIK#uLLT>s{y>BAnD0fb~wLz7n(!ksyYB~DR@vU!tX2z#jDD(&prcq zM?rQ(0!u1ueTyn&)O`i;o`M5yt?bDRdvOHRvc3WMK*5c=H%vip$VdbptNj2_K|w~e z0zWa@0)MP~)LH;kQjj=4n{{PB!r!#k(RKh;6zB*Qv0R1hGBl*vls^EfDY$pGk1ffZ z-Hw1DsS7|Y1^P$Rn6A_2+X%=G>jh9pflyuuo83?LAp-A?_Y3*AC-9Milv;Zh9`YZ( zdv(_V06tN0d2lY%H6MfTqU^B^;2 zF#Pf$e#atkGy?T~QUJbDU?Ca7p1N0FLEzD-i2%M+FsXJhe`WI@e+?#GW-@>#3bHDE znahNCHK>Bc$Eg6CDH!ng8GD(36t89yUd;gTgMulJWBA9#-uQAKl{_23PYT3mx-bQ` z*0X3x0cLXn{GvcDIG&l^tek-WOIrY-g@UD5Ch=Ql->eb%lDG&!D+S}Uo-q^O5BUBt zzKzwhk`Hn25^ay+vgyV=dA+ZF9l)_z3khnZ;=T2nydxTNkO>N zRpxkMxgP?GkJba|qCjJg5RcmO6CZq+J2nC6p`qH zqYAz8y8&2J(7Pa)wS5U+k3f>W9snB(ZpNjv^+%4)LttFA0RUSHj*k@O!SRQ2k4$c| zApkoHF8Jj!rI!QlqY4q%4+5~KU|3}gJ3Qz5WCYyK9|mxRg2Vs3xtZAQj*nKgoyPz; zP%wMrBevnu7Tf}!dF})NM+#<17qO^~x%gP>vp)skDg^@`A7wGd-uNvqU-L5noG8$_ zQp~!ejYZLfmSNtP_FX8Lk8q@11`0Q@O9x$heLQ2z560-@Gn00Jm5 zK3&Vs4qK#-z<{N<09>b_Fg%^r$)CgfY4sn`00JoxG917y)V*Gy3QwoT0SKaCho~^u zJT?6{24@lh1XD0Aq?tJg-7Z4F!r%^o5DFqQZn5@R#~l!`cS!>fNa#*N+(M43DZ}xov;S}u7iD8pvbMebUY)meI2nyo;UD@(=l6b8#eQW`M zNDB5Vy0V)Sv&W+$$$oqY;1&gW3v$>QVd*yrJV+`65JkbHoM%jOd|U(q1HU~65KVy~ zJC$AAnTS6n`&p#`Vkpq~WX;yD3hqS}r0UB6#8SY%gt4sIZ}H0!pVzzsaGQcekwkXo zlgmm}Ve5>y0OBY(-+GgUIx1%%ApGS$fOrZ777?s0EeY@Ar0!P&NT6WZ>Tq^$fwD5H z;McDfKq3X2dnNe9m+E)}ZruG5KoSLYG9FB+>D~a#ZgSP(!kVZkfZ3%1WJB3><&)Qo6+@;`P zVGT2poV*u(tQF=R0MaR_ZM@BXwl3I&fK|<302vf??ayQuQ_7SOh)?SVaE}5V@0naZ zZL>B4ET<1Z76n(#4l(c0tMUj`j_ZHp-!1-q3Y5EJSX{Ni8U&_|76Fh=!KI+}Z1sWN z;s^+j9Sk6cf@veGS+GnFehzd}R2)Dq19`fYW!Xdk zg%tF+X=ZbV9>Z4+H^0dM9#J6O99aU>byG*KoJF{#rK(zbtUdEj7N4}c_poD@^yRWj}D-AZFAzgei55QvzoCGC|4NLukzz!c-08c2;oaN1K zMgPDb>)qtV0G?8yH87uTwsZ196}pU;0(eHjqI3RC#pFgh0trHj07@x{`&h=VT#1xK zVBBI3;5h|rO`bEi2~qfAaLTxq0LmzE)yid71JXXD3iS(>0hCi9rs2=(GgJ>D(7kI7 zfEN@L9WP-a-s27;kYTe9z)K2T9*kvi6IxUd*cZAHz$*#_?!jz^LQ5wCA2)9X@R|Yz zw^TNx_`@Uw22555@P>jwU1wHXIeh>Ea~5w0@Rox9f1H?(QOF+zyhm#Rct^pO6(hM% z`ly=-R36_2;5`NLZ!DRA;QL1iG%V8v@PUGwufFVMTfl1sPW0&ksGvaqcn+I#;52^g z@#nIA04gciI?ay>bS0Lc3Uh;u08~-%F5nKk*YBk_0w0}>0aR1)q{D+XX>G-e6T_!R z0Mt-$X3;ICcIPX;$BnKv0Z>apyOJ$CYcCjuh7^-|5Bt zfF=r-=bm9bLJ@d-+$!t?pqYXe+iBeAp!hyChdERH0sNrA)^!={pHzhpzS&0u0sN#u zsRYAw5_Z4xojCOPgYt?}R-A5$Jkx3qUIc{Z}}! zvQZcClh+u{SO9GlEJ?Ox=f@c}p$aR8#sg@lz&aq0^-PI?VN<4$K@s#1L&hb$}5k>npEQL@ywhj04zml-`m%l3GK3JKy!!=Ed^jjK~uI9 z3m8^{dozVM$^lqYU{buFJ)3k1-{bbgy#ioE0ncz{S7+Y6frjL+_!fXI1wF10*ze?H zJqVn+@Bx4w1vOi}SOzl*L*T^VY5?{Wd~9%MJ*h!j2n_pJ3*ZU`8#*L;{nDA|5m52@ z1i*oUk!EqM-1D;t0>wRF060=$HoA;0jdERvK>tJE09>VD`Li z_o4J+1UiyB0k~11JMI#j9bIaRz^R8l0Ng1ka(Kad7VgJqoX|(1(0_OA9u!DjjAETL zRE1H6tLgm#cv5hCnLO`)vOEice&YuL@S@=0Y;RWRpT7tJIe{2}YZPn{d}0qI)65a5 z=n)6tO+iVY9%JKI%|$??VmJUF3Ze!rV3Vh<#_Ofn=c54lQBV*R#Pp;spQ8#hiX;K} zQ*h>30vi#hfY;v1lg0zMPJ#TnUu>G~S-ku)|1%Ll;C}&)Ud%f@&tCR`mGnL@a>x%fN%;lX8mD@oKF8l zpnRu1fCvgqvc9uO8>WaD+~%$I8b#87bBJ&OJKIy@WA;nUpp0AeX< zE6Zp5LKfnlv8u)<0JkYvryj=s*!c-ig^aK*0OBav`#6eKl-zxQfL_fu0Pz%5-B)9e z1W)iSt14I%Kmr95ed5?+NhMKKp>^s`0EraHyIx}*>-KCwfT`~WkVJuK>|>_y7*~gY zrH3AXWC}*h`Z0aC)A-<O)E1VFpe0%~xIt4~M zZm>~3D{K%@lQ092K|#aXNG3ce_Ywl!W z6K)S6mx3ExqS@T=6;DxxDcc?k0 z0-KS>Y}%)!NCe_CvjM!KV42n-Ci?Y`I|7rV^8mc2V6tr>+gLVf9Re=<9sqblLEJVg zX8lYCuPQg_KLYTUf&jrO=CsKPuPSS2mH>E1fx>rV_Uv}cSTv-O%1;5jrywnD7=JTs z#3BSnCq4)8fr5>r%-Nu>CcON)o%sSl1qCO(oYKh9B?X&Yce0WNM!2bF zKmR>|Y6@1{TQQyeq0wj#L&7Qm)KIWgVL6LB9go}CDPlDMYANuQ9m=l-xt>N9Wgde5GJa ze@oW8Eu#fhh%^2Ppn-y!S=Q{v?o#|%r)h61fJO?ckKAJRnWMj;3I&%t0DPlBb91z3~ZM}XJRZy1b0nkK&Sa~YbT_)OwK*Touc5?>@jg zMd?1_oB!_Ee^RhiC6)>AYr%_?%@aicv`}ztS}A*xU%D0zY1-Do09q+n;ya9&P4mI4 zN+}C*0BsboS?=u8mu*3)!rYA#06HjeS$deoINIUc?iK%00De=Te!qYj8XvxgD(F^A z0{BD0=P`-wlGP;K!5TVwJb=Fxl-l*So%gVxze8bBunBV~@VXS=HM(U9Kln*yMV z0<9Q-=2UwSUuN{ArvvDw;IY>+mbllf991}BISW7!1+(u2vdUQs`08C`CIg_C0$ZIZ zwq`}&3RGdU)qDVb6zqMwjrn-wTOqK@dm#YJL8L)zINyOSi0rz6z~=XI0IVpOYh=Y1 zjURy*Cv#im0a#NYEO2Juzn{au8goN+IRG09e2>So(v+R}XnlHF0Kk@liRXQoU*=%E zLos$&0$@kMz=y8v^dBQbH0f6{Dgf*$uw3zoZOicPL*R(dS^!rlP!w`zZBw1_yV9Kn z>j5}WAp80<6VonHMHP3D7=Skg8w-QkiLjUQ2rLmZ0pLSH_~ye*e#!HK>NE5fM5#Be_dto zSKq@6tincn03j4S>dIx}3Oe|SU#p)ZfE)h>C|a`Rs(LwSNa{u|075C4y7xA_VbC)Q zfrugQ0B%w+;LR1bX!}gOGF}+w1t5%qz2_FO#JAPYQ3Y2!9{}MLn2j)Ix$0t15GeTW z4}v*mHAy2ikw&27p@>1ed*JiXXCYn~P_OV32(=H{537yvO8XeJzFzYezG^TA?BJb>F2)D-Ho zyZ43hb5V1fBmi*~7!22AspAIW%e`4w3V?VDHlIGp4!P{8M_-(yX*z&J3Z{wJv!J`H z@Hf4rJQF|?1xFsZvzR}}_n-mmSY6ik<~W6=i}vOrQgDQLq{R$wD0_}IkZ0bCFyp*;n{RSYP0@GvGY=pfn zeo8Y!=LdiS3hGAbvh+>EZlNJHTD1UpK!InyCOg;Du7W_*t#$wpDae!Ez!LX~973S& z?H>Sz6sQmNU{hT)79()ycNc(16nH(G#pFa=eju=KO&@?F3g%ilunlTq9th;^77qJ& zKV3}0Gof2ddf&uY1Z1NJ0w|$ix5#c5KWG*nQbnvNfTt7;+@sEp{ZQJ8Dol(U0^k`1 z0oy8A%;3CY1O(#40hChEU-3G7_i2_Q0?rj90hCdo*}0fGJ!hE$FN@PdM`i}KmX%a?FRcV*8+052&pQC-M}G#ovLDg>UI0^k(|0k3zlGl!qz zmY15#bO5g@c)8b+#U>lz&ibWwvjM!N;7t1gw)@}}S2QFi)42fNQDC%q6$=^XgD>~d zeG34*r=YOw6FXhMeK4x9?bu=fA1DyYG+-6Ec6jNqDt;+|3JSJXcV;CTl4q3)fLaPZ zt=h%RBWK`G>HZc~0Cf~>-!OqI>9rq36#^4C0QgA3vbuZhj&U|Vr)x9R0DPk0o#0N@)1NuLc^jY~3a(CQB} z0`Q%JgR$a#<+hquRAHH{F@Pot@=m)lwQ-B^VqGNa2!Lh^TE>{OH@64km4L@*697Lb zs2}IaY6pGtM?(@xF$M68f`s9YjB5^AhrpagX8^QPP`9LvIUMf9>(-I8&I4$pATYy$ zy?TE7G^+5u>>_}63Wi_sXE`mRcpsOy-wHqn1!JVnu-uc4cx9XzU<=?k1y_HmFwM)= zMQBL##yJ4^LqT`cIkrNmcpd`U{!Re?QlMTb&LrM&{NQMzy&Hf|3X+T5SzEuXQ>en; zZchMR6wFhRWw!F=xa)9Xi4TBo3WWaNVM9k;jYbv5+xr9Pq2NupH`8~u8jpa~lpp}T z6dW4j%Tk+W;0?#Cb2k9=QLs2!g~bSqzCsng35Nr)6eSJXqoWL1>Y6%91fCte1;C1e zRsX!XS+(Kl9Rwbw#Q?DWFQ7}69o*C(|HhH%!gv5S6x^6=$xH(kqfmtv^+^D1DH!{E zISV)-TY-R_UMc`P3Z5-^%0AlH;$B5%jDUpsvn8q&}T9Myr1?M@g9H&1zI~}*z|2R(r8HeCY1m@DbVrs zWfhMG;7+2@-5LO16io2*V8y-4YfuHHp`QR;qd?y5G^@+mx*Y+_%r5}EDd-&<&a`AQ z@z&CD)HeV=6kM@)WV6Rg;#=0Z>pubbQgC7U6;|>#27eEdN7?}RQ6Ox%nklP!Z$)#M zUGy7(KLxsd3)y3-FUt|QR@DU{fPzNr?d)&b$d?EdZs-FLNP(rDCezg^#h1#CNa65* zcZxw2#16`5zB2#u|CBG$5&;lOLDkRsY)VxdzGW>ME(YKx1=2CW%tiAgUWO`7910+e zf@Q5gQvyIZ1$+0dWcAKx@Qqz`_-FtT6da75@y{}Jy9uiB(Q7P#ND2x! zdo%O9&bUpH(>Wf%EeiU1UT61$8t@6EwQv%EC<;uSMzcV}Gx%574cNPh zyYLp@!e<767z$G4_OgnPyYYK7-=@w15KF<#4MN;Gx+ejBN{a^118|#y^@~rj@;fpT z2y9K11rSF;$Eul3Z{bn=$h_DY*1x zIcs!KGC&pPKiUc)gM!E3Q<_%xTHF1fZCLkpT+q zknqdRXh>HAtpJozAQ`2{ltSFp5D<;B1@M@H1D`FK>WL&h1bnYt0q~T9h;;(iuCRO% z0=DlQ0X(BXQThtgx>Sh2>5g4407@wkd+~vlc8S!W3Kk*m0G?BDL24)K^q0raw4V<%`fHqmz}I&-0lcPQtHLT4(6{v`0#(x^0lcBWX!v5L z-{j?mz%d>T;4KBBkz3igxK(&Nkg?`AfOiy}jkv;s7FFSk@fe-}-~$B_4u_e)#*y7< zNI6H70aQ>R=_JqAXobE(plW<7fJzEdn)H}d?0@`j*|*qq096!pom6GZ0wsM=g(0n( z0IDf)@EFVVFKd?}5dJwEK+S&vuCthBufaJ4e*5MDsHGr%#wHdq@RtGtg>nx7)KPH# z;%a6u_v9o3YkeL8_((xQwKEgG)w>gcD<&lXK2Z?+$ABH~n12<4xgk#he5Rmr$vWog zS@;!!+{4cS)Kjo(A!FN@so}%B_uUHsUnppJY0R8kRc4|J1%0mpe5GL4)gx@io>`>` zG&Z~g&_Kcdm-4JbqW&5JlcrPxXr!P$U zzEj}-$AP8zOXH?mzlP5MnkeWpSj;ph2F^x9I@G@bKr;mjTYHwj?>84;`3!`=1NcFK zxQi9LQ5SFzRnQRp0Pu@~Q?UW8c%0A*1je0c0nkE$(GnB3`cxZ!4x|#=4xp8Sgmpuh z%a@NKs6xxkzW~}O_-?Y8{ap8@3W3@+-2mDt@MG3&b@<9n2>kuq3!sC7;d|xS@3@#| z1hPN&i}-f~{+ogWPD|OWvnF`+I$dNSfIk#;xE*J?-L~eaf@r}Y0DmbsS2Uho$*s~v zVDU#W0G$*hj$X%{!WZIQmTK)#09_PZN;<-piod|?rM`O-0J+`lFrs`T7h5Rb`V_%z=4A2%dauz z_=))HJ+Dj-fFlKKZ;Js6657X-U!Gq zPzB&d!Qg8a%>1|&K9=5I+5o_lf*6#P|&)22fMIB*bjjbJ^BCwDbUVsWacjyF$9b!?FSGPXY*|pykpE=6quO90XKtPXh?2;PVD2_PKKj-r|o4JPRO# z0;@mLY;Ecwe9Jnq>jHpC3UV(lVsFpuOQIpUAGi$Q76thhn_2v2uUQCeS!N9&iULc& z9c)NRCh`q=@9rr^&3S=M#keLe!U*$x0=C