(original) (raw)

%!PS-Adobe-2.0 %%Creator: dvips 5.516 Copyright 1986, 1993 Radical Eye Software %%Title: top.dvi %%CreationDate: Wed Sep 14 17:06:40 1994 %%Pages: 31 %%PageOrder: Ascend %%BoundingBox: 0 0 612 792 %%EndComments %DVIPSCommandLine: dvips -o top.ps top %DVIPSSource: TeX output 1994.09.14:1706 %%BeginProcSet: tex.pro /TeXDict 250 dict def TeXDict begin /N{def}def /B{bind def}N /S{exch}N /X{S N}B /TR{translate}N /isls false N /vsize 11 72 mul N /hsize 8.5 72 mul N /landplus90{false}def /@rigin{isls{[0 landplus90{1 -1}{-1 1} ifelse 0 0 0]concat}if 72 Resolution div 72 VResolution div neg scale isls{landplus90{VResolution 72 div vsize mul 0 exch}{Resolution -72 div hsize mul 0}ifelse TR}if Resolution VResolution vsize -72 div 1 add mul TR matrix currentmatrix dup dup 4 get round 4 exch put dup dup 5 get round 5 exch put setmatrix}N /@landscape{/isls true N}B /@manualfeed{ statusdict /manualfeed true put}B /@copies{/#copies X}B /FMat[1 0 0 -1 0 0]N /FBB[0 0 0 0]N /nn 0 N /IE 0 N /ctr 0 N /df-tail{/nn 8 dict N nn begin /FontType 3 N /FontMatrix fntrx N /FontBBox FBB N string /base X array /BitMaps X /BuildChar{CharBuilder}N /Encoding IE N end dup{/foo setfont}2 array copy cvx N load 0 nn put /ctr 0 N[}B /df{/sf 1 N /fntrx FMat N df-tail}B /dfs{div /sf X /fntrx[sf 0 0 sf neg 0 0]N df-tail}B /E{ pop nn dup definefont setfont}B /ch-width{ch-data dup length 5 sub get} B /ch-height{ch-data dup length 4 sub get}B /ch-xoff{128 ch-data dup length 3 sub get sub}B /ch-yoff{ch-data dup length 2 sub get 127 sub}B /ch-dx{ch-data dup length 1 sub get}B /ch-image{ch-data dup type /stringtype ne{ctr get /ctr ctr 1 add N}if}B /id 0 N /rw 0 N /rc 0 N /gp 0 N /cp 0 N /G 0 N /sf 0 N /CharBuilder{save 3 1 roll S dup /base get 2 index get S /BitMaps get S get /ch-data X pop /ctr 0 N ch-dx 0 ch-xoff ch-yoff ch-height sub ch-xoff ch-width add ch-yoff setcachedevice ch-width ch-height true[1 0 0 -1 -.1 ch-xoff sub ch-yoff .1 add]{ ch-image}imagemask restore}B /D{/cc X dup type /stringtype ne{]}if nn /base get cc ctr put nn /BitMaps get S ctr S sf 1 ne{dup dup length 1 sub dup 2 index S get sf div put}if put /ctr ctr 1 add N}B /I{cc 1 add D }B /bop{userdict /bop-hook known{bop-hook}if /SI save N @rigin 0 0 moveto /V matrix currentmatrix dup 1 get dup mul exch 0 get dup mul add .99 lt{/QV}{/RV}ifelse load def pop pop}N /eop{SI restore showpage userdict /eop-hook known{eop-hook}if}N /@start{userdict /start-hook known{start-hook}if pop /VResolution X /Resolution X 1000 div /DVImag X /IE 256 array N 0 1 255{IE S 1 string dup 0 3 index put cvn put}for 65781.76 div /vsize X 65781.76 div /hsize X}N /p{show}N /RMat[1 0 0 -1 0 0]N /BDot 260 string N /rulex 0 N /ruley 0 N /v{/ruley X /rulex X V}B /V {}B /RV statusdict begin /product where{pop product dup length 7 ge{0 7 getinterval dup(Display)eq exch 0 4 getinterval(NeXT)eq or}{pop false} ifelse}{false}ifelse end{{gsave TR -.1 -.1 TR 1 1 scale rulex ruley false RMat{BDot}imagemask grestore}}{{gsave TR -.1 -.1 TR rulex ruley scale 1 1 false RMat{BDot}imagemask grestore}}ifelse B /QV{gsave transform round exch round exch itransform moveto rulex 0 rlineto 0 ruley neg rlineto rulex neg 0 rlineto fill grestore}B /a{moveto}B /delta 0 N /tail{dup /delta X 0 rmoveto}B /M{S p delta add tail}B /b{S p tail} B /c{-4 M}B /d{-3 M}B /e{-2 M}B /f{-1 M}B /g{0 M}B /h{1 M}B /i{2 M}B /j{ 3 M}B /k{4 M}B /w{0 rmoveto}B /l{p -4 w}B /m{p -3 w}B /n{p -2 w}B /o{p -1 w}B /q{p 1 w}B /r{p 2 w}B /s{p 3 w}B /t{p 4 w}B /x{0 S rmoveto}B /y{ 3 2 roll p a}B /bos{/SS save N}B /eos{SS restore}B end %%EndProcSet TeXDict begin 40258431 52099146 1000 300 300 (/tmp_mnt/home/u1/tommi/papers/stoch/final/top.dvi) @start /Fa 2 49 df0 D<081C1C3838383070706060C0C0060D7E 8D09>48 D E /Fb 1 22 df<00F800000000FE000000001F800000000F800000000FC000 000007C000000007E000000007E000000003E000000003F000000003F000000001F00000 0001F800000001F800000000F800000000FC000000007C000000007E000000007E000000 003E000000003F000000003F000000001F000000001F800000001F800000000F80000000 0FC00000000FC00000001FC00000003FE000000077E0000000E3E0000001C3F000000383 F000000701F800000F01F800001E00F800003C00FC00007800FC0000F0007C0001E0007E 0003C0007E0007C0003E000F80003F001F00003F003E00001F007C00001F80F800001F80 F800000FC070000007C022327CB12A>21 D E /Fc 1 51 df<7FFFFFC0FFFFFFE0C00000 60C0000060C0000060C0000060C0000060C0000060C0000060C0000060C0000060C00000 60C0000060C0000060C0000060C0000060C0000060C0000060C0000060C0000060C00000 60C0000060C0000060C0000060C0000060FFFFFFE0FFFFFFE01B1B7B9E25>50 D E /Fd 1 22 df<07800001C00000E00000E00000F00000700000700000700000380000 3800003800003C00001C00001C00001E00000E00001E00003F0000670000C70001878003 03800703800E03801C03C03801C07001C0E001E06000E0131D7E9C18>21 D E /Fe 31 122 df<004000800100020006000C000C0018001800300030007000600060 006000E000E000E000E000E000E000E000E000E000E000E000E000600060006000700030 003000180018000C000C00060002000100008000400A2A7D9E10>40 D<800040002000100018000C000C000600060003000300038001800180018001C001C001 C001C001C001C001C001C001C001C001C001C0018001800180038003000300060006000C 000C00180010002000400080000A2A7E9E10>I<60F0F0701010101020204080040C7C83 0C>44 DI<60F0F06004047C830C>I68 D73 D<003F800000E0E0000380380007001C000E000E001C0007003C00078038000380780003 C0780003C0700001C0F00001E0F00001E0F00001E0F00001E0F00001E0F00001E0F00001 E0F00001E0700001C0780003C0780003C0380003803C0E07801C1107000E208E0007205C 0003A0780000F0E020003FE0200000602000003060000038E000003FC000003FC000001F 8000000F001B257E9C20>81 D<7FFFFFC0700F01C0600F00C0400F0040400F0040C00F00 20800F0020800F0020800F0020000F0000000F0000000F0000000F0000000F0000000F00 00000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F00 00000F0000000F0000001F800003FFFC001B1C7F9B1E>84 D87 D<1FC000307000783800781C00301C00001C00001C0001FC000F1C00381C00701C00601C 00E01C40E01C40E01C40603C40304E801F870012127E9115>97 DI<07E00C301878307870306000E000E000E000E000E000E0 0060007004300418080C3007C00E127E9112>I<003F0000070000070000070000070000 070000070000070000070000070000070003E7000C1700180F00300700700700600700E0 0700E00700E00700E00700E00700E00700600700700700300700180F000C370007C7E013 1D7E9C17>I<03E00C301818300C700E6006E006FFFEE000E000E000E000600070023002 18040C1803E00F127F9112>I<00F8018C071E061E0E0C0E000E000E000E000E000E00FF E00E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E007FE00F 1D809C0D>I<00038003C4C00C38C01C3880181800381C00381C00381C00381C00181800 1C38000C300013C0001000003000001800001FF8001FFF001FFF803003806001C0C000C0 C000C0C000C06001803003001C0E0007F800121C7F9215>II<18003C003C0018000000000000000000000000000000FC001C 001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C00FF80091D7F 9C0C>I108 DII<03F0000E1C00180600300300700380600180E001C0 E001C0E001C0E001C0E001C0E001C06001807003803003001806000E1C0003F00012127F 9115>II114 D<1F9030704030C010C010 E010F8007F803FE00FF000F880388018C018C018E010D0608FC00D127F9110>I<040004 00040004000C000C001C003C00FFE01C001C001C001C001C001C001C001C001C001C101C 101C101C101C100C100E2003C00C1A7F9910>IIII121 D E /Ff 1 50 df<0C001C00EC000C000C 000C000C000C000C000C000C000C000C000C000C000C000C000C00FFC00A137D9211>49 D E /Fg 17 115 df<00180030006000C001C00380070007000E001E001C003C003C003C 007800780078007800F800F000F000F000F000F000F000F000F000F000F000F000F80078 007800780078003C003C003C001C001E000E0007000700038001C000C00060003000180D 317BA416>40 DI<387CFEFEFE7C3807077C8610>46 D<00180000780001F800FFF800FFF80001F8 0001F80001F80001F80001F80001F80001F80001F80001F80001F80001F80001F80001F8 0001F80001F80001F80001F80001F80001F80001F80001F80001F80001F80001F80001F8 007FFFE07FFFE013207C9F1C>49 D<03FC000FFF003C1FC07007E07C07F0FE03F0FE03F8 FE03F8FE01F87C01F83803F80003F80003F00003F00007E00007C0000F80001F00003E00 00380000700000E01801C0180380180700180E00380FFFF01FFFF03FFFF07FFFF0FFFFF0 FFFFF015207D9F1C>I<00FE0007FFC00F07E01E03F03F03F03F81F83F81F83F81F81F03 F81F03F00003F00003E00007C0001F8001FE0001FF000007C00001F00001F80000FC0000 FC3C00FE7E00FEFF00FEFF00FEFF00FEFF00FC7E01FC7801F81E07F00FFFC001FE001720 7E9F1C>I<387CFEFEFE7C380000000000000000387CFEFEFE7C3807167C9510>58 D76 D80 D<7FFFFFFF807FFFFFFF807E03F80F807803F807807003F803806003F80180E003F801C0 E003F801C0C003F800C0C003F800C0C003F800C0C003F800C00003F800000003F8000000 03F800000003F800000003F800000003F800000003F800000003F800000003F800000003 F800000003F800000003F800000003F800000003F800000003F800000003F800000003F8 00000003F800000003F800000003F8000003FFFFF80003FFFFF80022227EA127>84 D<07FC001FFF803F07C03F03E03F01E03F01F01E01F00001F00001F0003FF003FDF01FC1 F03F01F07E01F0FC01F0FC01F0FC01F0FC01F07E02F07E0CF81FF87F07E03F18167E951B >97 D<00FE0007FF800F87C01E01E03E01F07C00F07C00F8FC00F8FC00F8FFFFF8FFFFF8 FC0000FC0000FC00007C00007C00007E00003E00181F00300FC07003FFC000FF0015167E 951A>101 D<003F8000FFC001E3E003C7E007C7E00F87E00F83C00F80000F80000F8000 0F80000F80000F8000FFFC00FFFC000F80000F80000F80000F80000F80000F80000F8000 0F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80007FF800 7FF80013237FA211>I104 D109 D<00FE0007FFC00F83E01E00F03E00F87C007C7C007C7C007C FC007EFC007EFC007EFC007EFC007EFC007EFC007E7C007C7C007C3E00F81F01F00F83E0 07FFC000FE0017167E951C>111 D114 D E /Fh 3 51 df<00C00000C00000C00000C0 0000C00000C00000C00000C00000C000FFFF80FFFF8000C00000C00000C00000C00000C0 0000C00000C00000C00000C00011147E8F17>43 D<0C003C00CC000C000C000C000C000C 000C000C000C000C000C000C000C00FF8009107E8F0F>49 D<1F00618040C08060C06000 60006000C00180030006000C00102020207FC0FFC00B107F8F0F>I E /Fi 2 117 df<71F09A189C18981818183030303030323062606460380F0B7E8A13> 110 D<08181818FF30303030606062646438080F7E8E0C>116 D E /Fj 7 112 df<0000700001F00003C0000780000E00001C0000380000700000700000 F00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E00000E00000E00000E00001C00001C00001C00003800007000006 00000E0000380000700000C000007000003800000E000006000007000003800001C00001 C00001C00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E00000E00000E00000E00000E00000F00000700000700000380000 1C00000E000007800003C00001F000007014637B811F>26 DI80 DI88 D<0000E00003E0000F8000 1E00003C0000700000700000E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00001C00001C0000380000700000E00003C0000F00000F000003C00000E 000007000003800001C00001C00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E000007000007000003C00001E00000F800003E00000E0134A7C81 1C>110 DI E /Fk 9 62 df<001800001800003C00003C 00004E00004E000087000087000103800303C00201C00601E00400E00C00F00800701000 7810003820003C20001C40001E7FFFFEFFFFFFFFFFFF18177E961D>1 D<0102040C1818303070606060E0E0E0E0E0E0E0E0E0E060606070303018180C04020108 227D980E>40 D<8040203018180C0C0E060606070707070707070707070606060E0C0C18 183020408008227E980E>I<003000003000003000003000003000003000003000003000 003000003000003000FFFFFCFFFFFC003000003000003000003000003000003000003000 00300000300000300000300016187E931B>43 D<07C018303018701C600C600CE00EE00E E00EE00EE00EE00EE00EE00EE00E600C600C701C30181C7007C00F157F9412>48 D<03000700FF000700070007000700070007000700070007000700070007000700070007 00070007007FF00C157E9412>I<0F8030E040708030C038E03840380038007000700060 00C00180030006000C08080810183FF07FF0FFF00D157E9412>I<00300030007000F001 F001700270047008701870107020704070C070FFFE0070007000700070007003FE0F157F 9412>52 D61 D E /Fl 5 51 df0 D<020002000200C218F2783AE00F800F803AE0F278C2180200020002000D0E7E8E12>3 D<0000010000000080000000800000004000000020FFFFFFFCFFFFFFFC00000020000000 400000008000000080000001001E0C7E8D23>33 D<0F8007C019E01C2020703010401840 08C00C8004800780048007000480038004800780048004C00C400860082030381010E01E 600F8007C01E0E7E8D23>49 D<01FF8007FF800E00001800003000006000006000006000 00C00000C00000FFFF80FFFF80C00000C000006000006000006000003000001800000E00 0007FF8001FF8011167D9218>I E /Fm 23 121 df<07C0000C30401010402018806018 80C00C80C00D00C00E00800E00800C00C01C00C02C0060C4803F0300120E7E8D17>11 D<07C01C00300060006000FF00C000C000C000C000C000400030C01F000A0E7E8D0E>15 D<0E00030003800180018001C000C000C000600060007000300030007800980118020C04 0C080C180630066007C00310177E9615>21 D<0402000C06000C06000C0600180C00180C 00180C00180C003018003018803018803038807859006F8E00600000600000C00000C000 00C0000080000011147E8D15>I<07FFF800E00E00E00700E00300E00301C00301C00701 C00701C00E03803C03FFF003FFF003803C07001C07000E07000E07000E0E001C0E001C0E 00380E00701C01E0FFFF0018177F961B>66 D<07F007F800F000C000B8008000B8008000 9C0080011C0100011E0100010E0100010E01000207020002070200020382000203820004 01C4000401C4000400E4000400E4000800780008007800080038000800380018001000FE 0010001D177F961C>78 D<003E1000C1A00100E00200600600600C00400C00400E00000F 000007E00007FC0001FE00003F0000078000038000038020018040030040030060060060 0400D8180087E00014177E9615>83 D85 D87 D<071018F0307060706060C060C060C06080C080C480C4C1C446C838700E0E7E8D13>97 D<07C01C20301060106020FFC0C000C000C000C000C010402060C01F000C0E7E8D10> 101 D<1F0006000600060006000C000C000C000C0018F01B181C08180838183018301830 306030603160616062C022C03C10177E9614>104 D<0300038003000000000000000000 000000001C002400460046008C000C0018001800180031003100320032001C0009177F96 0C>I<00180038001000000000000000000000000001C002200430043008600060006000 6000C000C000C000C001800180018001806300E300C60078000D1D80960E>I<1F000600 0600060006000C000C000C000C00181C1866188E190C32003C003F00318060C060C460C4 60C8C0C8C0700F177E9612>I<383C1E0044C6630047028100460301008E0703000C0603 000C0603000C060600180C0600180C0620180C0C20180C0C4030180440301807801B0E7F 8D1F>109 D<383C0044C6004702004602008E06000C06000C06000C0C00180C00180C40 181840181880300880300F00120E7F8D15>I<1C3C22462382230346030603060306030C 060C060C0C0C081A3019E018001800300030003000FC001014808D12>112 D<07C00C201870187038001E000FC003E000606060E060C0C0C1803F000C0E7E8D10> 115 D<030003000600060006000600FFC00C000C000C0018001800180018003000308030 80310031001E000A147F930D>I<1C0200260600460600460600860C000C0C000C0C000C 0C001818001818801818801838800C5900078E00110E7F8D14>I<1C04260E4606460686 040C040C040C0418081808181018100C6007800F0E7F8D11>I<0F1F0011A18020C38020 C300418000018000018000018000030000030200C30200E70400C5080078F000110E7F8D 14>120 D E /Fn 16 119 df77 D<01FF800007FFF0000F81F8 001FC07E001FC07E001FC03F000F803F8007003F8000003F8000003F8000003F80000FFF 8000FFFF8007FC3F800FE03F803F803F803F003F807F003F80FE003F80FE003F80FE003F 80FE003F807E007F807F00DF803F839FFC0FFF0FFC01FC03FC1E1B7E9A21>97 DI<001FF80000FFFE0003F01F0007E03F800FC03F801F803F803F801F007F800E007F00 00007F000000FF000000FF000000FF000000FF000000FF000000FF000000FF0000007F00 00007F0000007F8000003F8001C01F8001C00FC0038007E0070003F01E0000FFFC00001F E0001A1B7E9A1F>I<00003FF80000003FF80000003FF800000003F800000003F8000000 03F800000003F800000003F800000003F800000003F800000003F800000003F800000003 F800000003F800000003F800001FE3F80000FFFBF80003F03FF80007E00FF8000FC007F8 001F8003F8003F8003F8007F0003F8007F0003F8007F0003F800FF0003F800FF0003F800 FF0003F800FF0003F800FF0003F800FF0003F800FF0003F8007F0003F8007F0003F8007F 0003F8003F8003F8001F8003F8000F8007F80007C00FF80003F03BFF8000FFF3FF80003F C3FF80212A7EA926>I<003FE00001FFF80003F07E0007C01F000F801F801F800F803F80 0FC07F000FC07F0007C07F0007E0FF0007E0FF0007E0FFFFFFE0FFFFFFE0FF000000FF00 0000FF0000007F0000007F0000007F0000003F8000E01F8000E00FC001C007E0038003F8 1F0000FFFE00001FF0001B1B7E9A20>I<07000FC01FE03FE03FE03FE01FE00FC0070000 00000000000000000000000000FFE0FFE0FFE00FE00FE00FE00FE00FE00FE00FE00FE00F E00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE0FFFEFFFEFFFE0F2B7DAA14 >105 D107 DII< FFC07F0000FFC1FFC000FFC787E0000FCE03F0000FD803F0000FD803F8000FF003F8000F F003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE0 03F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003 F8000FE003F8000FE003F800FFFE3FFF80FFFE3FFF80FFFE3FFF80211B7D9A26>I<003F E00001FFFC0003F07E000FC01F801F800FC03F800FE03F0007E07F0007F07F0007F07F00 07F0FF0007F8FF0007F8FF0007F8FF0007F8FF0007F8FF0007F8FF0007F8FF0007F87F00 07F07F0007F03F800FE03F800FE01F800FC00FC01F8007F07F0001FFFC00003FE0001D1B 7E9A22>II114 D<03FE300FFFF01E03F03800F0700070F00070F00070F80070FC0000 FFE0007FFE007FFF803FFFE01FFFF007FFF800FFF80003FC0000FC60007CE0003CF0003C F00038F80038FC0070FF01E0F7FFC0C1FF00161B7E9A1B>I118 D E /Fo 46 121 df<003F000000E180000380C020070060400E0070401C0070403C0070803C0038 80780039007800390078003A00F0003A00F0003C00F0003800F000380070003800700078 003000B800380338401C1C188007E00F001B157E941F>11 D<00007C0000018300000201 8000040180000801C0001001C0002001C0002001C0004001C00040038000800380008003 000080070001000E000107FC0001083800010FDC0002000E0002000E0002000F0002000F 0004000F0004000F0004000F0004000F0008001E0008001E0008001C0008003C00140038 00140070001400E0001201C00021838000207C0000200000002000000040000000400000 004000000040000000800000008000000080000000800000001A2D7EA21C>I<01F00107 F8010FFC021FFC02380E046002044003088001088001080000900000900000A00000A000 00A00000C00000C00000C000008000008000008000018000018000018000030000030000 030000030000060000060000060000040000040018207F9419>I<000F80001860006020 00401000C00001800001800001800001800001C00001E00001F00000F800003C00003E00 00EF000387000703800E03801C01803C01803C0180780180780180780180F00100F00100 F00300F00200700600700400300C003808001C300007C00014237EA216>I<007E038007 000E001C00380078007000FFF8F000F000E000E000E000E000E0007000700038061C1807 E00F157D9414>I<000F000031800060C000E0C001C0E00180E00380E00700E00700E00F 00E00E00E01E00E01E00E01C01E03C01E03C01E03C01E07FFFC07803C07803C07803C070 0780F00780F00700F00F00F00E00F00E00E01C00601C0060380070300070600030C00011 80000F000013237EA217>18 D<03800000E00000F0000070000070000078000038000038 00003C00001C00001C00001C00001E00000E00000E00000F000007000007000007800007 80000F80001B800031C00021C00041C000C1E00180E00300E00600F00C00701C00703800 78700038E00038C0001C16237DA21C>21 D<00C00C01C01C01C01C038038038038038038 0380380700700700700700700700700E00E00E00E00E00E00E00E11E01C21E01C21E03C2 1E04C43F08C439F038380000380000700000700000700000700000E00000E00000E00000 E00000C0000018207E941D>I<1F0000402380008021C0010001C0010001E0020000E004 0000F008000070100000702000007820000038400000388000003D0000001E0000001C00 00001E0000000E0000001E0000002E0000004F0000008700000107000002078000020380 00040380000803C0001001C0002001E0004000E1008000720080003C001A1F7E941F>31 D<04000060080000F0080000F81000007810000038200000182000001840030010400300 10400600104006001040060020C0040020C00C0040C00C00C0C01E0180E07607807FE7FF 007FE7FE003F83FC001F01F0001D1580941E>33 D<70F8F8F87005057C840E>58 D<70F8FCFC7404040404080810102040060F7C840E>I<0000001800000078000001E000 00078000001E00000078000003E000000F8000003C000000F0000003C000000F0000003C 000000F0000000F00000003C0000000F00000003C0000000F00000003C0000000F800000 03E0000000780000001E0000000780000001E000000078000000181D1C7C9926>I<0000 80000180000180000300000300000300000600000600000600000C00000C00000C000018 0000180000180000300000300000300000600000600000600000C00000C00000C0000180 000180000180000180000300000300000300000600000600000600000C00000C00000C00 00180000180000180000300000300000300000600000600000600000C00000C00000C000 0011317DA418>II<00007F00800003808100000E0063000038 0027000070001F0000E0000E0001C0000E000380000E000700000E000F000004000E0000 04001E000004003C000004003C00000800780000000078000000007800000000F0000000 00F000000000F000000000F000000000F000000000E000000000E000002000E000002000 E000004000E000004000F00000800070000080007000010000380002000018000400001C 0008000006003000000381C0000000FE00000021247DA223>67 D<007FFFFFC000078003 C000078000C000078000C0000F0000C0000F0000C0000F000080000F000080001E000080 001E000080001E008080001E008000003C010000003C010000003C030000003C07000000 7FFE000000780600000078060000007806000000F004000000F004000000F004010000F0 00020001E000020001E000020001E000040001E0000C0003C000080003C000180003C000 300003C000700007C003F000FFFFFFE00022227EA124>69 D<007FFFFFC000078003C000 078000C000078000C0000F0000C0000F0000C0000F000080000F000080001E000080001E 000080001E008080001E008000003C010000003C010000003C030000003C070000007FFE 000000780600000078060000007806000000F004000000F004000000F004000000F00000 0001E000000001E000000001E000000001E000000003C000000003C000000003C0000000 03C000000007C0000000FFFE00000022227EA120>I<00007F00400003C0C080000E0021 80001C0013800070000F8000E000070001C0000700038000070007000007000F00000200 0E000002001E000002003C000002003C00000400780000000078000000007800000000F0 00000000F000000000F000000000F000000000F0003FFF00E00000F000E00000F000E000 00F000E00001E000F00001E000F00001E000700001E000700003C000380003C000180007 C0000C0009C00006001180000380E08000007F00000022247DA226>I<007FC00001FF00 07C00003E00007C00005E00007C00005E00009E0000BC00009E0000BC00009E00013C000 09E00023C00011E00027800011E00047800011E00047800011E00087800021E0010F0000 20F0010F000020F0020F000020F0040F000040F0041E000040F0081E000040F0081E0000 40F0101E000080F0203C00008078203C00008078403C00008078803C0001007880780001 007900780001007900780001007A00780002007C00F00002007C00F00002003800F00006 003800F0000F003001F000FFE0203FFF0030227EA12F>77 D<007FC003FF0007C0007800 07C000600005E000200009E000400009E000400008F000400008F000400010F800800010 780080001078008000103C008000203C010000203E010000201E010000201E010000400F 020000400F020000400F020000400782000080078400008007C400008003C400008003C4 00010001E800010001E800010001F800010000F800020000F00002000070000200007000 06000070000F00002000FFE000200028227EA127>I<007FFFF0000007801C000007800F 000007800700000F000380000F000380000F000380000F000380001E000780001E000780 001E000780001E000F00003C000F00003C001E00003C003C00003C007000007801E00000 7FFF00000078000000007800000000F000000000F000000000F000000000F000000001E0 00000001E000000001E000000001E000000003C000000003C000000003C000000003C000 000007C0000000FFFC00000021227EA11F>80 D<00007F00000381C0000E006000380038 0070003800E0001C01C0001E0380000E0700000E0F00000F0E00000F1E00000F3C00000F 3C00000F7800000F7800000F7800000FF000001EF000001EF000001EF000001CF000003C E000003CE0000078E0000078E00000F0E00000E0F00001E0F01E03C07021038078408700 38408E001C40B8000E40F00007C1C02000FE80200000802000018040000180C00001C080 0001C3800001FF000000FF000000FE0000007800202D7DA227>I<007FFFE0000007803C 000007800E000007800700000F000780000F000380000F0003C0000F0003C0001E000780 001E000780001E000780001E000F00003C001E00003C003C00003C007000003C01C00000 7FFE00000078078000007801C000007801E00000F000F00000F000F00000F000F00000F0 00F00001E001E00001E001E00001E001E00001E001E00003C003C00003C003C04003C003 C04003C001C08007C001C080FFFC00E3000000003C0022237EA125>I<0003F010000E0C 2000180260002001E0004000E000C000C0008000C0018000C0018000C003800080038000 80038000000380000003C0000001F0000001FE000000FFE000007FF000001FF8000001FC 0000003C0000001C0000000E0000000E0000000E0020000C0020000C0020000C00200008 0060001800600010007000200070004000C8008000C603000081FC00001C247DA21E>I< 1FFFFFFE1E01E00E1801E0063001E0062003C0062003C0064003C0044003C00440078004 800780048007800400078000000F0000000F0000000F0000000F0000001E0000001E0000 001E0000001E0000003C0000003C0000003C0000003C0000007800000078000000780000 0078000000F0000000F0000000F0000000F0000001F00000FFFFC0001F227EA11D>I<3F FE01FF8003C0003C0003C000300003C00010000780002000078000200007800020000780 0020000F000040000F000040000F000040000F000040001E000080001E000080001E0000 80001E000080003C000100003C000100003C000100003C00010000780002000078000200 00780002000078000200007000040000F000040000F00008000070000800007000100000 7000200000380040000038008000001C01000000060600000001F800000021237DA121> III<007FFC03FF0007E000F80007C000E00003C000800003E001000001E002000001 F006000001F00C000000F018000000F81000000078200000007C400000007C800000003D 000000003E000000001E000000001F000000001F000000002F000000006F80000000C780 00000187C000000103C000000203C000000403E000000801E000001001F000002000F000 004000F800008000F80001800078000300007C000F8000FC00FFE007FFC028227FA128> II<0078 6001C4E00302E00601C00E01C01C01C03C01C0380380780380780380780380F00700F007 00F00700F00708F00E10700E10701E1030262018C6200F01C015157E941A>97 D<003F0000E0800380C00701C00E03C01C03C03C00003C0000780000780000780000F000 00F00000F00000F000007000407000403001803802001C1C0007E00012157E9415>99 D<00E000E001E000C00000000000000000000000000000000000001E0023004380438083 808380870007000E000E000E001C001C0038003820384070407040308031001E000B227E A111>105 D<0000E00001E00001E00000C0000000000000000000000000000000000000 000000000000000000001E00002300004380008380010380010380010380000700000700 000700000700000E00000E00000E00000E00001C00001C00001C00001C00003800003800 00380000380000700000700000700070E000F0C000F180006300003E0000132C81A114> I<00F0000FE00000E00000E00000E00001C00001C00001C00001C0000380000380000380 000380000700000700F00703080704380E08780E10780E20300E40001C80001F00001FC0 001C7000383800383800381C00381C10703820703820703820701840E00C806007001523 7DA219>I<3C07E01F00461830618047201880C087401D00E087801E00E087801C00E087 001C00E00E003801C00E003801C00E003801C00E003801C01C007003801C007003801C00 7007001C007007043800E007083800E00E083800E00E083800E006107001C006203000C0 03C026157E942B>109 D<3C07C04618604720308740388780388700388700380E00700E 00700E00700E00701C00E01C00E01C01C01C01C13801C238038238038238018470018830 00F018157E941D>I<03C0F004631C04740E08780E08700708700708700F00E00F00E00F 00E00F00E00F01C01E01C01E01C01E01C03C03803803803803C07003C0E0072180071E00 0700000700000E00000E00000E00000E00001C00001C00001C0000FFC000181F819418> 112 D<3C0F004630C04741C08783C08783C08701808700000E00000E00000E00000E0000 1C00001C00001C00001C000038000038000038000038000070000030000012157E9416> 114 D<007E0000810003008002018006038006030006000007000007F80003FE0001FF00 003F00000780000380700380F00300F00300E002004004003018000FE00011157E9417> I<006000E000E000E000E001C001C001C001C00380FFFC0380038007000700070007000E 000E000E000E001C001C001C001C08381038103820182018C007000E1F7F9E12>I<1E00 182300384380384380708380708380708700700700E00E00E00E00E00E00E01C01C01C01 C01C01C01C01C21C03841C03841C07840C09880E118803E07017157E941C>I<1E00C023 01E04381F04380F08380708380308700300700200E00200E00200E00201C00401C00401C 00801C00801C01001C01001C02000C040006080003F00014157E9418>I<1E0018182300 383C4380383E4380701E8380700E83807006870070060700E0040E00E0040E00E0040E00 E0041C01C0081C01C0081C01C0081C01C0101C01C0101C01C0201C03C0400C04C0C00708 E10001F03E001F157E9423>I<01E0F006310C081A1C101A3C201C3C201C18201C000038 0000380000380000380000700000700000700000700860E010F0E010F0E020E170404230 803C1F0016157E941C>I E /Fp 55 123 df<00000FC0F8000030718E0000E0F31E0000 C0F71E0001C0660C0001800E000003800E000003800E000003800E000007001C00000700 1C000007001C000007001C000007001C0000FFFFFFC0000E003800000E003800000E0038 00000E003800001C007000001C007000001C007000001C007000001C007000001C00E000 003800E000003800E000003800E000003800E000003801C000007001C000007001C00000 7001C000007001C000006003800000E003800000E003800000E003000000C003000001C0 070000718E060000F19E0C0000F31E180000620C3000003C07C00000272D82A21E>11 D<00000FE0000030180000E01C0001C03C0001803C000380380003800000038000000700 0000070000000700000007000000070000000E000000FFFFE0000E00E0000E00E0000E01 C0001C01C0001C01C0001C01C0001C0380001C0380003803800038038000380700003807 00003807000070070800700E1000700E1000700E1000700E2000E0062000E003C000E000 0000E0000000C0000001C0000001C0000071800000F1800000F3000000620000003C0000 001E2D82A21B>I<00000FF01FE000003838601800006079C01C0000C07B803C0001C033 003C0001C00700380003800700000003800700000003800E00000003800E00000007000E 00000007000E00000007000E00000007001C000000FFFFFFFFE0000E001C00E0000E001C 00E0000E001C01C0000E003801C0000E003801C0001C003801C0001C00380380001C0038 0380001C00700380001C00700380001C00700700003800700700003800700700003800E0 0708003800E00E10003800E00E10007000E00E10007000E00E20007001C00620007001C0 03C0006001C0000000E00180000000E00380000000C00380000000C00300000071C70300 0000F18F06000000F10F0C0000006206180000003C03E00000002E2D82A22B>14 D<00008000010000020000040000080000100000300000600000C00000C0000180000300 000300000600000600000E00000C00001C00001800001800003800003000003000007000 00700000600000600000E00000E00000E00000C00000C00000C00000C00000C00000C000 00C00000C00000C00000C00000C00000C00000C000004000006000006000002000003000 00100000080000113278A414>40 D<000800000400000600000200000300000300000100 000180000180000180000180000180000180000180000180000180000180000180000180 000180000380000380000380000300000300000700000700000600000600000E00000C00 000C00001C0000180000380000300000300000600000600000C000018000018000030000 060000040000080000100000200000400000800000113280A414>I<0E1E1E1E1E020204 04080810204080070F7D840F>44 DI<70F8F8F0E005057A 840F>I<0001000300030006001E002E03CE001C001C001C001C00380038003800380070 00700070007000E000E000E000E001C001C001C001C003800380038003800780FFFC1021 7AA019>49 D<000FC000106000603800801800801C01001C02201E02101E04101E04101E 04101E08203C08203C0840380840780880F00700E00001C0000300000600001800002000 00C0000100000200000400100800301000202000605F80C063FFC040FF80807F00801E00 17227CA019>I<000FC000307000C01801001C02001C04000C04401C08201C08201C0820 1C08403808C0380700700000600001C000070000FC000007000003800003800001C00001 C00001C00003C06003C0F003C0F00380E00780800700800E00801C0040380020F0001F80 0016227BA019>I<0000180000380000380000700000700000700000E00000E00000E000 00C00001C0000180000380000300000300000600000600000C00000C0000180000100000 31800061C00043800083800183800303800207000407000807003FC700403E00800FF000 0E00000E00001C00001C00001C00001C00003800003800003800003000152B7EA019>I< 000FC000306000401000801801000803000C03000C06001807001807001007003007C060 07E0C003F18001FE0000FC0000FE00033F00061F800C07C01803C03001C06001C06000C0 C000C0C000C0C00080C00180C00100C00200C006006008003030000FC00016227BA019> 56 D<07000F800F800F000E000000000000000000000000000000000000000000000070 00F800F800F000E00009157A940F>58 D<0000030000000300000007000000070000000F 0000000F0000001F0000002F0000002F0000004F0000004F800000878000008780000107 800002078000020780000407800004078000080780000807800010078000300780002007 80007FFF80004007C0008007C0008003C0010003C0030003C0020003C0040003C0040003 C00C0003C03C0007C0FF003FFC1E237DA224>65 D<00FFFFE0000F0038000F001C000F00 1E001E000E001E000F001E000F001E000F003C000E003C001E003C001E003C003C007800 78007800F0007801E00078078000FFFF8000F001E000F000F000F0007801E0007801E000 3801E0003C01E0003C03C0007803C0007803C0007803C000F0078000F0078001E0078003 C0078007000F801E00FFFFF00020227DA122>I<00007F00800003808100000E00630000 380027000070001F0000E0000E0001C0000E000380000E000700000E000F000004000E00 0004001E000004003C000004003C00000800780000000078000000007800000000F00000 0000F000000000F000000000F000000000F000000000E000000000E000002000E0000020 00E000004000E000004000F0000080007000008000700001000038000200001800040000 1C0008000006003000000381C0000000FE000000212479A223>I<00FFFFF000000F003C 00000F000E00000F000700001E000380001E000380001E0001C0001E0001C0003C0001C0 003C0001E0003C0001E0003C0001E000780001E000780001E000780001E000780001E000 F00003C000F00003C000F00003C000F00003C001E000078001E000078001E000070001E0 000F0003C0000E0003C0001C0003C0003C0003C00038000780007000078000E000078001 C00007800700000F801C0000FFFFF0000023227DA125>I<00FFFFFF000F000F000F0003 000F0003001E0003001E0003001E0002001E0002003C0002003C0002003C0102003C0100 00780200007802000078060000780E0000FFFC0000F00C0000F00C0000F00C0001E00800 01E0080001E0080001E0000003C0000003C0000003C0000003C000000780000007800000 07800000078000000F800000FFFC000020227DA120>70 D<00FFF87FFC000F000780000F 000780000F000780001E000F00001E000F00001E000F00001E000F00003C001E00003C00 1E00003C001E00003C001E000078003C000078003C000078003C000078003C0000FFFFF8 0000F000780000F000780000F000780001E000F00001E000F00001E000F00001E000F000 03C001E00003C001E00003C001E00003C001E000078003C000078003C000078003C00007 8003C0000F8007C000FFF87FFC0026227DA124>72 D<00FFF8000F00000F00000F00001E 00001E00001E00001E00003C00003C00003C00003C0000780000780000780000780000F0 0000F00000F00000F00001E00001E00001E00001E00003C00003C00003C00003C0000780 000780000780000780000F8000FFF80015227DA113>I<0007FFC000003C0000003C0000 003C00000078000000780000007800000078000000F0000000F0000000F0000000F00000 01E0000001E0000001E0000001E0000003C0000003C0000003C0000003C0000007800000 0780000007800000078000000F0000000F0000380F0000780F0000F81E0000F81E0000F0 3C0000403800004070000021E000001F8000001A237CA11A>I<00FFFC00000F8000000F 0000000F0000001E0000001E0000001E0000001E0000003C0000003C0000003C0000003C 00000078000000780000007800000078000000F0000000F0000000F0000000F0000001E0 000001E0000001E0002001E0002003C0004003C0004003C0008003C00080078001800780 01000780030007800F000F803E00FFFFFE001B227DA11F>76 D<00FF800007FC000F8000 0F80000F80001780000F80001780001780002F000013C0002F000013C0004F000013C000 8F000023C0009E000023C0011E000023C0011E000023C0021E000043C0043C000043C004 3C000043C0083C000041E0083C000081E01078000081E02078000081E02078000081E040 78000101E040F0000101E080F0000101E100F0000101E100F0000200F201E0000200F201 E0000200F401E0000200F801E0000400F803C0000400F003C0000400F003C0000C00E003 C0001E00C007C000FFC0C07FFC002E227DA12C>I<00FF000FFC000F8001E0000F800180 000FC000800013C001000013C001000011E001000011E001000021E002000020F0020000 20F002000020F0020000407804000040780400004078040000403C040000803C08000080 3E080000801E080000801E080001001F100001000F100001000F10000100079000020007 A000020007A000020003E000020003E000040003C000040001C000040001C0000C0001C0 001E00008000FFC000800026227DA124>I<0000FE0000078380000C00E0003800700070 003800E0003801C0001C0380001C0700001C0F00001E1E00001E1C00001E3C00001E3C00 001E7800001E7800001E7800001EF000003CF000003CF000003CF0000078F0000078E000 0078E00000F0E00000F0E00001E0E00001C0F00003C0F00007807000070078000E003800 1C001C0038000E00E0000703800001FC00001F2479A225>I<00FFFFE0000F0038000F00 1E000F000E001E0007001E0007001E0007001E0007003C000F003C000F003C000F003C00 1E0078001E0078003C00780078007800E000F003C000FFFE0000F0000000F0000001E000 0001E0000001E0000001E0000003C0000003C0000003C0000003C0000007800000078000 0007800000078000000F800000FFF8000020227DA121>I<0000FE0000078380000C00E0 003800700070003800E0003801C0003C0380001C0700001C0F00001E1E00001E1E00001E 3C00001E3C00001E7800001E7800001E7800001EF000003CF000003CF000003CF0000038 F0000078E0000078E00000F0E00000F0E00001E0E00001C0E00003C0F01E038070610700 70810E0038811C001C8138000E81E0000783804001FD0040000100400001808000018180 00018100000387000001FE000001FE000001FC000000F0001F2D79A225>I<0001F02000 0E0C40001802C0003001C0006001C000E0018000C0018001C0018001C0018003C0010003 C0010003C0000003C0000003E0000001F8000001FF000000FFE000007FF000001FF80000 03FC0000007C0000003C0000001E0000001E0000001E0020001C0020001C0020001C0020 0018006000380060003000700060007000C000C8018000C607000081FC00001B247DA21B >83 D<1FFFFFF81E03C0381803C0183003C0182007801820078018400780104007801040 0F0010800F0010800F0010000F0000001E0000001E0000001E0000001E0000003C000000 3C0000003C0000003C00000078000000780000007800000078000000F0000000F0000000 F0000000F0000001E0000001E0000001E0000001E0000003E00000FFFF00001D2277A123 >I<00F8C00185C00705C00E03800E03801C03803C0380380700780700780700780700F0 0E00F00E00F00E00F00E10F01C20701C20703C20305C40308C400F078014157B9419>97 D<03C03F8003800380038007000700070007000E000E000E000E001C001CF81D0C1E0E3C 0638073807380F700F700F700F700FE01EE01EE01EE03CE038E038607060E031C01F0010 237BA216>I<007E0001C1000301800703800E07801C07803C0000380000780000780000 780000F00000F00000F00000F00000F00100700100700200300C001830000FC00011157B 9416>I<00003C0003F80000380000380000380000700000700000700000700000E00000 E00000E00000E00001C000F9C00185C00705C00E03800E03801C03803C03803807007807 00780700780700F00E00F00E00F00E00F00E10F01C20701C20703C20305C40308C400F07 8016237BA219>I<00F803840E021C023C0238027804F018FFE0F000F000E000E000E000 E000E002E0026004701830600F800F157A9416>I<00003E0000470000CF00018F000186 000380000380000380000700000700000700000700000700000E0000FFF0000E00000E00 000E00001C00001C00001C00001C00001C00003800003800003800003800003800007000 00700000700000700000700000E00000E00000E00000E00000C00001C00001C000718000 F18000F300006200003C0000182D82A20F>I<001F180030B800E0B801C07001C0700380 700780700700E00F00E00F00E00F00E01E01C01E01C01E01C01E01C01E03800E03800E07 80060B8006170001E700000700000700000E00000E00000E00701C00F01800F0300060E0 003F8000151F7E9416>I<00F0000FE00000E00000E00000E00001C00001C00001C00001 C000038000038000038000038000070000071F0007218007C0C00F00E00F00E00E00E00E 00E01C01C01C01C01C01C01C01C0380380380380380380380704700708700E08700E1070 0610E006206003C016237DA219>I<00C001E001C001C000000000000000000000000000 0000001C002300430043008700870087000E000E001C001C001C00380038003840708070 807080710032001C000B217BA00F>I<00F0000FE00000E00000E00000E00001C00001C0 0001C00001C0000380000380000380000380000700000701E0070210070C700E10F00E10 F00E20600E40001D80001E00001FC0001C7000383800383800381C00381C207038407038 40703840701880E01880600F0014237DA216>107 D<01E01FC001C001C001C003800380 0380038007000700070007000E000E000E000E001C001C001C001C003800380038003800 7000700070007100E200E200E200E200640038000B237CA20C>I<1C0F80F8002610C10C 00476066060087807807008780780700870070070087007007000E00E00E000E00E00E00 0E00E00E000E00E00E001C01C01C001C01C01C001C01C01C001C01C03820380380384038 038070403803807080380380308070070031003003001E0023157B9428>I<1C0F002631 C04740C08780E08780E08700E08700E00E01C00E01C00E01C00E01C01C03801C03801C03 801C0704380708380E08380E103806107006203003C016157B941B>I<007E0001C30003 81800701C00E01C01C01E03C01E03801E07801E07801E07801E0F003C0F003C0F00380F0 0780700700700E00700C0030180018700007C00013157B9419>I<01C1F002621804741C 08780C08700E08700E08701E00E01E00E01E00E01E00E01E01C03C01C03C01C03C01C078 03807003807003C0E003C1C0072380071E000700000700000E00000E00000E00000E0000 1C00001C00001C0000FFC000171F7F9419>I<00F8400184C00705C00E03800E03801C03 803C0380380700780700780700780700F00E00F00E00F00E00F00E00F01C00701C00703C 00305C0030B8000F380000380000380000700000700000700000700000E00000E00000E0 000FFE00121F7B9416>I<1C1F002620804741C08783C08703C08701808700000E00000E 00000E00000E00001C00001C00001C00001C000038000038000038000038000070000030 000012157B9415>I<00FC000183000200800401800C03800C03000C00000F00000FF000 07FC0003FE00003E00000F00000700700700F00600F00600E004004008002030001FC000 11157D9414>I<00C001C001C001C001C003800380038003800700FFF8070007000E000E 000E000E001C001C001C001C003800380038003810702070207040708031001E000D1F7C 9E10>I<1E00602300E04380E04381C08381C08701C08701C00703800E03800E03800E03 801C07001C07001C07001C07081C0E10180E101C0E101C1E200C262007C3C015157B941A >I<1E03802307C04387C04383C08381C08700C08700C00700800E00800E00800E00801C 01001C01001C01001C02001C02001C04001C08001C08000C300003C00012157B9416>I< 1E0060E02300E1F04380E1F04381C0F08381C0708701C0308701C030070380200E038020 0E0380200E0380201C0700401C0700401C0700401C0700801C0700801C0701001C0F0100 0C0F020006138C0003E0F0001C157B9420>I<03C1E0046210083470103CF02038F02038 6020380000700000700000700000700000E00000E00000E00000E02061C040F1C040F1C0 80E2C100446200383C0014157D9416>I<1E00302300704380704380E08380E08700E087 00E00701C00E01C00E01C00E01C01C03801C03801C03801C03801C07001C07001C07001C 0F000C3E0003CE00000E00000E00001C00601C00F03800F03000E0600080C0004380003E 0000141F7B9418>I<01E02003F06007F8C0041F80080100080200000400000800001000 0020000040000080000100000200000400800801001003003F060061FC0040F800807000 13157D9414>I E /Fq 1 22 df<07000001C00000E00000E00000F00000700000700000 7800003800003800003C00001C00001C00001E00000E00000E00000F00000700000F0000 378000638000C38001C3C00381C00701C00E01E01C00E03800E07000F0F00070E00070C0 003815207D9F1B>21 D E /Fr 41 122 df<0020004000800100020006000C000C001800 18003000300030007000600060006000E000E000E000E000E000E000E000E000E000E000 E000E0006000600060007000300030003000180018000C000C0006000200010000800040 00200B2E7DA112>40 D<800040002000100008000C000600060003000300018001800180 01C000C000C000C000E000E000E000E000E000E000E000E000E000E000E000E000C000C0 00C001C001800180018003000300060006000C00080010002000400080000B2E7DA112> I<70F8FCFC74040404080810102040060E7C840D>44 DI<70F8 F8F87005057C840D>I<018003800F80F380038003800380038003800380038003800380 03800380038003800380038003800380038003800380038003800380038007C0FFFE0F1E 7C9D17>49 D<03F0000C0C00100600300300200180600180600180600180700180780300 3E03003F06001FC8000FF00003F80007FC000C7E00103F00300F806003804001C0C001C0 C000C0C000C0C000C0C000806001802001001002000C0C0003F000121F7E9D17>56 D<03F0000E18001C0C00380600380700700700700380F00380F00380F003C0F003C0F003 C0F003C0F003C07007C07007C03807C0180BC00E13C003E3C00003800003800003800007 00300700780600780E00700C002018001070000FC000121F7E9D17>I68 D73 D77 D80 D<001F800000F0F00001C038 0007801E000F000F000E0007001E0007803C0003C03C0003C07C0003E07C0003E0780001 E0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001 F0780001E0780001E07C0003E03C0003C03C0F03C01E1087800E20C7000F20CF0007A0FE 0001E0F80000F0F010001FF01000007010000078300000387000003FF000003FE000001F E000000FC0000007801C297D9F23>II<07E0800C1980100780300380600180600180E00180E00080E00080E000 80F00000F000007800007F00003FF0001FFC000FFE0003FF00001F800007800003C00003 C00001C08001C08001C08001C08001C0C00180C00380E00300F00600CE0C0081F8001221 7D9F19>I<7FFFFFE0780F01E0600F0060400F0020400F0020C00F0030800F0010800F00 10800F0010800F0010000F0000000F0000000F0000000F0000000F0000000F0000000F00 00000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F00 00000F0000000F0000000F0000001F800007FFFE001C1F7E9E21>I87 D<1FE000303000781800781C00300E00000E00000E00000E00 00FE00078E001E0E00380E00780E00F00E10F00E10F00E10F01E10781E103867200F83C0 14147E9317>97 D<0E0000FE00000E00000E00000E00000E00000E00000E00000E00000E 00000E00000E00000E3E000EC3800F01C00F00E00E00E00E00700E00700E00780E00780E 00780E00780E00780E00780E00700E00700E00E00F00E00D01C00CC300083E0015207F9F 19>I<03F80E0C1C1E381E380C70007000F000F000F000F000F000F00070007000380138 011C020E0C03F010147E9314>I<000380003F8000038000038000038000038000038000 038000038000038000038000038003E380061B801C0780380380380380700380700380F0 0380F00380F00380F00380F00380F003807003807003803803803807801C07800E1B8003 E3F815207E9F19>I<03F0000E1C001C0E00380700380700700700700380F00380F00380 FFFF80F00000F00000F000007000007000003800801800800C010007060001F80011147F 9314>I<007C00C6018F038F07060700070007000700070007000700FFF0070007000700 0700070007000700070007000700070007000700070007000700070007007FF01020809F 0E>I<0000E003E3300E3C301C1C30380E00780F00780F00780F00780F00780F00380E00 1C1C001E380033E0002000002000003000003000003FFE001FFF800FFFC03001E0600070 C00030C00030C00030C000306000603000C01C038003FC00141F7F9417>I<0E0000FE00 000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E3E000E43 000E81800F01C00F01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01 C00E01C00E01C00E01C00E01C00E01C0FFE7FC16207F9F19>I<1C003E003E003E001C00 0000000000000000000000000E007E000E000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E000E00FFC00A1F809E0C>I<0E0000FE00000E00000E00000E 00000E00000E00000E00000E00000E00000E00000E00000E0FF00E03C00E03000E02000E 04000E08000E10000E30000E70000EF8000F38000E1C000E1E000E0E000E07000E07800E 03800E03C00E03E0FFCFF815207F9F18>107 D<0E00FE000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E00FFE00B20809F0C>I<0E1F01F000FE618618000E81C81C000F00 F00E000F00F00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E0 0E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E 000E00E00E00FFE7FE7FE023147F9326>I<0E3E00FE43000E81800F01C00F01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C0FFE7FC16147F9319>I<01F800070E001C03803801C03801C07000E07000E0F000 F0F000F0F000F0F000F0F000F0F000F07000E07000E03801C03801C01C0380070E0001F8 0014147F9317>I<0E3E00FEC3800F01C00F00E00E00E00E00F00E00700E00780E00780E 00780E00780E00780E00780E00700E00F00E00E00F01E00F01C00EC3000E3E000E00000E 00000E00000E00000E00000E00000E00000E0000FFE000151D7F9319>I<03E080061980 1C05803C0780380380780380700380F00380F00380F00380F00380F00380F00380700380 7803803803803807801C0B800E138003E380000380000380000380000380000380000380 000380000380003FF8151D7E9318>I<0E78FE8C0F1E0F1E0F0C0E000E000E000E000E00 0E000E000E000E000E000E000E000E000E00FFE00F147F9312>I<1F9030704030C010C0 10C010E00078007F803FE00FF00070803880188018C018C018E030D0608F800D147E9312 >I<020002000200060006000E000E003E00FFF80E000E000E000E000E000E000E000E00 0E000E000E000E080E080E080E080E080610031001E00D1C7F9B12>I<0E01C0FE1FC00E 01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E 01C00E01C00E03C00603C0030DC001F1FC16147F9319>III<7FC3 FC0F01E00701C007018003810001C20000E40000EC00007800003800003C00007C00004E 000087000107000303800201C00601E01E01E0FF07FE1714809318>II E /Fs 7 117 df<0000E000000000E000000001F000 000001F000000001F000000003F800000003F800000006FC00000006FC0000000EFE0000 000C7E0000000C7E000000183F000000183F000000303F800000301F800000701FC00000 600FC00000600FC00000C007E00000FFFFE00001FFFFF000018003F000018003F0000300 01F800030001F800060001FC00060000FC000E0000FE00FFE00FFFE0FFE00FFFE0231F7E 9E28>65 D<07FC001FFF003F0F803F07C03F03E03F03E00C03E00003E0007FE007FBE01F 03E03C03E07C03E0F803E0F803E0F803E0FC05E07E0DE03FF8FE0FE07E17147F9319>97 DI<01FE0007 FF801F0FC03E0FC03E0FC07C0FC07C0300FC0000FC0000FC0000FC0000FC0000FC00007C 00007E00003E00603F00C01F81C007FF0001FC0013147E9317>I114 D<0FE63FFE701E600EE006E006 F800FFC07FF83FFC1FFE03FE001FC007C007E007F006F81EFFFCC7F010147E9315>I<01 800180018003800380038007800F803F80FFFCFFFC0F800F800F800F800F800F800F800F 800F800F800F860F860F860F860F8607CC03F801F00F1D7F9C14>I E /Ft 83 125 df<000060000000006000000000F000000000F000000001F80000000178 000000027C000000023C000000043E000000041E000000081F000000080F000000100F80 000010078000002007C000002003C000004003E000004001E000008001F000008000F000 010000F80001000078000200007C000200003C000400003E000400001E000800001F0008 00000F001000000F80100000078020000007C020000003C07FFFFFFFE07FFFFFFFE0FFFF FFFFF024237EA229>1 D<001FE00000F03C0001C00E00078007800F0003C01F0003E01E 0001E03E0001F03C0000F07C0000F87C0000F87C0000F87C0000F87C0000F87C0000F87C 0000F83C0000F03E0001F03E0001F01E0001E01E0001E00E0001C00F0003C00700038003 000300038007000180060081800604808004044080040840C00C08404008087FC00FF83F C00FF03FC00FF01E237EA223>10 D<001F83E000706E3000C07C780180F8780380F07807 0070000700700007007000070070000700700007007000070070000700700007007000FF FFFFC0070070000700700007007000070070000700700007007000070070000700700007 007000070070000700700007007000070070000700700007007000070070000700700007 007000070078007FE3FF801D2380A21C>I<001FC0000070200000C01000018038000380 780007007800070030000700000007000000070000000700000007000000070000000700 0000FFFFF800070078000700380007003800070038000700380007003800070038000700 380007003800070038000700380007003800070038000700380007003800070038000700 380007003800070038007FE1FF80192380A21B>I<000FC07F00007031C08000E00B0040 01801E00E003803E01E007003C01E007001C00C007001C000007001C000007001C000007 001C000007001C000007001C000007001C0000FFFFFFFFE007001C01E007001C00E00700 1C00E007001C00E007001C00E007001C00E007001C00E007001C00E007001C00E007001C 00E007001C00E007001C00E007001C00E007001C00E007001C00E007001C00E007001C00 E007001C00E007001C00E07FF1FFCFFE272380A229>14 D22 D<7038F87CFC7EFC7E743A04020402040204020804080410081008201040200F0F7EA218 >34 D<003C000000006200000000C2000000018100000001810000000381000000038100 000003810000000381000000038200000003820000000384000000038800000001C80000 0001D000000001E003FF8001C0007C0000E000380001E000300001F00020000270004000 04700040000838008000183C008000301C010000701E020000700E020000F007040000F0 07880000F003880000F001D00100F000E0010078007003003800B802003C031C04000E0C 0E0C0003F003F00021257EA326>38 D<70F8FCFC7404040404080810102040060F7CA20E >I<00200040008001000300060004000C000C0018001800300030003000700060006000 6000E000E000E000E000E000E000E000E000E000E000E000E000E000E000600060006000 7000300030003000180018000C000C0004000600030001000080004000200B327CA413> I<800040002000100018000C000400060006000300030001800180018001C000C000C000 C000E000E000E000E000E000E000E000E000E000E000E000E000E000E000C000C000C001 C0018001800180030003000600060004000C00180010002000400080000B327DA413>I< 008000008000008000008000008000C08180E08380388E000C980003E00001C00003E000 0C9800388E00E08380C0818000800000800000800000800000800011157DA418>I<0001 800000018000000180000001800000018000000180000001800000018000000180000001 8000000180000001800000018000000180000001800000018000FFFFFFFEFFFFFFFE0001 800000018000000180000001800000018000000180000001800000018000000180000001 80000001800000018000000180000001800000018000000180001F227D9C26>I<70F8FC FC7404040404080810102040060F7C840E>II<70F8F8F8700505 7C840E>I<01F000071C000C06001803003803803803807001C07001C07001C07001C0F0 01E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F0 01E0F001E07001C07001C07001C07803C03803803803801C07000C0600071C0001F00013 227EA018>48 D<008003800F80F380038003800380038003800380038003800380038003 80038003800380038003800380038003800380038003800380038003800380038007C0FF FE0F217CA018>I<03F0000C1C001007002007804003C04003C08003E0F003E0F801E0F8 01E0F801E02003E00003E00003C00003C0000780000700000E00001C0000180000300000 600000C0000180000100000200200400200800201800603000403FFFC07FFFC0FFFFC013 217EA018>I<03F8000C1E001007002007804007C07807C07803C07807C03807C0000780 000780000700000F00000E0000380003F000001C00000F000007800007800003C00003C0 0003E02003E07003E0F803E0F803E0F003C04003C0400780200780100F000C1C0003F000 13227EA018>I<000200000600000E00000E00001E00001E00002E00004E00004E00008E 00008E00010E00020E00020E00040E00040E00080E00100E00100E00200E00200E00400E 00800E00FFFFF8000E00000E00000E00000E00000E00000E00000E00001F0001FFF01521 7FA018>I<1000801E07001FFF001FFE001FF80013E00010000010000010000010000010 000010000010F800130E001407001803801003800001C00001C00001E00001E00001E000 01E07001E0F001E0F001E0E001C08001C04003C04003802007001006000C1C0003F00013 227EA018>I<007E0001C1000300800601C00E03C01C03C0180180380000380000780000 700000700000F0F800F30C00F40600F40300F80380F801C0F001C0F001E0F001E0F001E0 F001E0F001E07001E07001E07001E03801C03801C01803801C03000C0600070C0001F000 13227EA018>I<4000006000007FFFE07FFFC07FFFC0400080C001008001008002008002 0000040000080000080000100000300000200000600000600000600000E00000C00000C0 0001C00001C00001C00001C00003C00003C00003C00003C00003C00003C00003C00003C0 0001800013237DA118>I<01F800060E000803001001802001802000C06000C06000C060 00C07000C07801803E01003F02001FC4000FF80003F80003FC00067F00083F80100F8030 07C06001C06000E0C000E0C00060C00060C00060C000606000406000C03000801803000E 0E0003F00013227EA018>I<01F000060C000C0600180700380380700380700380F001C0 F001C0F001C0F001E0F001E0F001E0F001E0F001E07001E07003E03803E01805E00C05E0 0619E003E1E00001C00001C00001C0000380000380300300780700780600700C00201800 1030000FC00013227EA018>I<70F8F8F870000000000000000000000070F8F8F8700515 7C940E>I<70F8F8F870000000000000000000000070F8F8F87808080808101010204040 051F7C940E>I61 D<000180000001800000 0180000003C0000003C0000003C0000005E0000005E000000DF0000008F0000008F00000 10F800001078000010780000203C0000203C0000203C0000401E0000401E0000401E0000 800F0000800F0000FFFF000100078001000780030007C0020003C0020003C0040003E004 0001E0040001E00C0000F00C0000F03E0001F8FF800FFF20237EA225>65 DI< 0007E0100038183000E0063001C00170038000F0070000F00E0000701E0000701C000030 3C0000303C0000307C0000107800001078000010F8000000F8000000F8000000F8000000 F8000000F8000000F8000000F800000078000000780000107C0000103C0000103C000010 1C0000201E0000200E000040070000400380008001C0010000E0020000381C000007E000 1C247DA223>IIII<0007F008003C0C1800E0021801C001B8038000F8070000780F0000381E000038 1E0000183C0000183C0000187C0000087800000878000008F8000000F8000000F8000000 F8000000F8000000F8000000F8000000F8001FFF780000F8780000787C0000783C000078 3C0000781E0000781E0000780F00007807000078038000B801C000B800E00318003C0C08 0007F00020247DA226>III<03FFF0001F00000F00000F00000F00000F00000F00000F 00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F 00000F00000F00000F00000F00000F00000F00700F00F80F00F80F00F80E00F01E00401C 0020380018700007C00014237EA119>IIIII<000FE00000783C0000E00E0003C00780078003C00F0001E00E0000 E01E0000F03C0000783C0000787C00007C7C00007C7800003C7800003CF800003EF80000 3EF800003EF800003EF800003EF800003EF800003EF800003EF800003E7800003C7C0000 7C7C00007C3C0000783E0000F81E0000F00F0001E00F0001E0078003C003C0078000E00E 0000783C00000FE0001F247DA226>II<000FE00000783C0000E00E0003C00780078003C00F 0001E00E0000E01E0000F03E0000F83C0000787C00007C7C00007C7800003C7800003CF8 00003EF800003EF800003EF800003EF800003EF800003EF800003EF800003EF800003E78 00003C7C00007C7C00007C3C0000783C0000781E0380F00E0420E00F0801E0078813C003 C8178000E80E00007C3C02000FEC0200000C0200000C0200000E0600000F0E000007FC00 0007FC000007F8000003F8000001E01F2D7DA226>II<03F0200C0C601802603001E0 7000E0600060E00060E00060E00020E00020E00020F00000F000007800007F00003FF000 1FFE000FFF0003FF80003FC00007E00001E00000F00000F0000070800070800070800070 800070C00060C00060E000C0F000C0C80180C6070081FC0014247DA21B>I<7FFFFFF878 07807860078018400780084007800840078008C007800C80078004800780048007800480 078004000780000007800000078000000780000007800000078000000780000007800000 078000000780000007800000078000000780000007800000078000000780000007800000 078000000780000007800000078000000FC00003FFFF001E227EA123>IIII< FFF0007FC01F80001F000F80000C00078000080007C000180003E000100001E000200001 F000200000F000400000F800C000007C008000003C010000003E010000001E020000001F 040000000F84000000078800000007D800000003D000000003E000000001E000000001E0 00000001E000000001E000000001E000000001E000000001E000000001E000000001E000 000001E000000001E000000001E000000003E00000003FFF000022227FA125>89 D91 D<0804100820102010402040208040 804080408040B85CFC7EFC7E7C3E381C0F0F7AA218>II<0FE0001838003C0C003C0E0018070000070000070000070000FF0007C7 001E07003C0700780700700700F00708F00708F00708F00F087817083C23900FC1E01515 7E9418>97 D<0E0000FE00001E00000E00000E00000E00000E00000E00000E00000E0000 0E00000E00000E00000E00000E1F000E61C00E80600F00300E00380E003C0E001C0E001E 0E001E0E001E0E001E0E001E0E001E0E001E0E001C0E003C0E00380F00700C80600C41C0 083F0017237FA21B>I<01FE000703000C07801C0780380300780000700000F00000F000 00F00000F00000F00000F00000F000007000007800403800401C00800C010007060001F8 0012157E9416>I<0000E0000FE00001E00000E00000E00000E00000E00000E00000E000 00E00000E00000E00000E00000E001F8E00704E00C02E01C01E03800E07800E07000E0F0 00E0F000E0F000E0F000E0F000E0F000E0F000E07000E07800E03800E01801E00C02E007 0CF001F0FE17237EA21B>I<01FC000707000C03801C01C03801C07801E07000E0F000E0 FFFFE0F00000F00000F00000F00000F000007000007800203800201C00400E0080070300 00FC0013157F9416>I<003C00C6018F038F030F07000700070007000700070007000700 0700FFF80700070007000700070007000700070007000700070007000700070007000700 0700070007807FF8102380A20F>I<00007001F198071E180E0E181C07001C07003C0780 3C07803C07803C07801C07001C07000E0E000F1C0019F000100000100000180000180000 1FFE000FFFC00FFFE03800F0600030400018C00018C00018C000186000306000303800E0 0E038003FE0015217F9518>I<0E0000FE00001E00000E00000E00000E00000E00000E00 000E00000E00000E00000E00000E00000E00000E1F800E60C00E80E00F00700F00700E00 700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00 700E00700E0070FFE7FF18237FA21B>I<1C001E003E001E001C00000000000000000000 000000000000000E00FE001E000E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E000E00FFC00A227FA10E>I<01C003E003E003E001C0000000000000 0000000000000000000001E00FE001E000E000E000E000E000E000E000E000E000E000E0 00E000E000E000E000E000E000E000E000E000E000E000E000E060E0F0C0F18061803E00 0B2C82A10F>I<0E0000FE00001E00000E00000E00000E00000E00000E00000E00000E00 000E00000E00000E00000E00000E03FC0E01F00E01C00E01800E02000E04000E08000E10 000E38000EF8000F1C000E1E000E0E000E07000E07800E03C00E01C00E01E00E00F00E00 F8FFE3FE17237FA21A>I<0E00FE001E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E00FFE00B237FA20E>I<0E1FC07F00FE60E183801E807201C00F003C00E0 0F003C00E00E003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E 003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E00 3800E00E003800E0FFE3FF8FFE27157F942A>I<0E1F80FE60C01E80E00F00700F00700E 00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E 00700E00700E0070FFE7FF18157F941B>I<01FC000707000C01801800C03800E0700070 700070F00078F00078F00078F00078F00078F00078F000787000707800F03800E01C01C0 0E038007070001FC0015157F9418>I<0E1F00FE61C00E80600F00700E00380E003C0E00 1C0E001E0E001E0E001E0E001E0E001E0E001E0E001E0E003C0E003C0E00380F00700E80 E00E41C00E3F000E00000E00000E00000E00000E00000E00000E00000E00000E0000FFE0 00171F7F941B>I<01F8200704600E02601C01603801E07800E07800E0F000E0F000E0F0 00E0F000E0F000E0F000E0F000E07000E07800E03801E01C01E00C02E0070CE001F0E000 00E00000E00000E00000E00000E00000E00000E00000E00000E0000FFE171F7E941A>I< 0E3CFE461E8F0F0F0F060F000E000E000E000E000E000E000E000E000E000E000E000E00 0E000F00FFF010157F9413>I<0F8830786018C018C008C008E008F0007F803FE00FF001 F8003C801C800C800CC00CC008E018D0308FC00E157E9413>I<02000200020002000600 060006000E001E003E00FFF80E000E000E000E000E000E000E000E000E000E000E000E04 0E040E040E040E040E040708030801F00E1F7F9E13>I<0E0070FE07F01E00F00E00700E 00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E 00F00E00F006017003827800FC7F18157F941B>IIII< FFC1FE1E00780E00300E00200E002007004007004003808003808003808001C10001C100 00E20000E20000E200007400007400003800003800003800001000001000002000002000 002000004000F04000F08000F180004300003C0000171F7F941A>I<3FFFC03803803007 80200700600E00401C00403C0040380000700000E00001E00001C0000380400700400F00 400E00C01C0080380080780180700780FFFF8012157F9416>I124 D E /Fu 32 122 df<78FCFCFCFC7806067B8511>46 D<000018000000001800 00000018000000003C000000003C000000003C000000007E000000007E00000000FF0000 00009F000000009F000000011F800000010F800000010F8000000207C000000207C00000 0207C000000403E000000403E000000403E000000801F000000801F000001801F8000010 00F800001000F800002000FC000020007C00003FFFFC00007FFFFE000040003E00004000 3E000080001F000080001F000080001F000100000F800100000F800100000F8002000007 C007000007C01F80000FE0FFF000FFFFFFF000FFFF282A7EA92D>65 DI<0000FF 00100007FFE030001FC07830003E000C7000F80006F001F00003F003E00001F007C00000 F00F800000700F800000701F000000303F000000303E000000303E000000107E00000010 7E000000107C00000000FC00000000FC00000000FC00000000FC00000000FC00000000FC 00000000FC00000000FC00000000FC000000007C000000007E000000007E000000103E00 0000103E000000103F000000101F000000200F800000200F8000006007C000004003E000 008001F000018000F8000300003E000E00001FC038000007FFE0000000FF8000242B7DA9 2B>II73 D<03FFFF03FFFF0007E00003E00003 E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003 E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003 E00003E00003E03003E07803E0FC03E0FC03E0FC03C0F807C0400780200F00300E000C3C 0003F000182A7DA81F>I77 D80 D<00FE010003FF83000F81E3001E0037003C001F0038 000F007800070070000700F0000300F0000300F0000300F0000100F8000100F8000100FC 0000007C0000007F0000003FE000001FFF00000FFFE00007FFF80003FFFC00007FFE0000 07FF0000007F0000001F8000000F80000007C0000007C0800003C0800003C0800003C080 0003C0C00003C0C0000380C0000380E0000780F0000700F8000E00EE001C00C3C07800C1 FFF000803FC0001A2B7DA921>83 D<7FFFFFFFF87FFFFFFFF87C007C00F870007C003860 007C001840007C000840007C0008C0007C000CC0007C000C80007C000480007C00048000 7C000480007C000480007C000400007C000000007C000000007C000000007C000000007C 000000007C000000007C000000007C000000007C000000007C000000007C000000007C00 0000007C000000007C000000007C000000007C000000007C000000007C000000007C0000 00007C000000007C000000007C000000007C000000007C00000000FE000000FFFFFE0000 FFFFFE0026297EA82B>II<01FC00000E0780001001C0003C00E0003E00F0003E007800 1C00780008007800000078000000780000007800007FF80003E078000F8078001F007800 3E0078007C00780078007820F8007820F8007820F8007820F800F8207C00F8203C013C40 1F063FC007F80F001B1A7E991E>97 D<007F8001C0700780080F003C1E007C3C007C3C00 387C0010780000F80000F80000F80000F80000F80000F80000F80000F800007800007C00 003C00043C00041E00080F001007802001C0C0007F00161A7E991B>99 D<00000F000001FF000001FF0000001F0000000F0000000F0000000F0000000F0000000F 0000000F0000000F0000000F0000000F0000000F0000000F0000000F00003F0F0001C0CF 0003802F000F001F001E001F001C000F003C000F007C000F0078000F0078000F00F8000F 00F8000F00F8000F00F8000F00F8000F00F8000F00F8000F0078000F0078000F003C000F 003C000F001E001F000E002F0007004F8001C18FF8007E0FF81D2A7EA921>I<007E0003 C3800700E00E00F01C00703C00783C003878003C78003CF8003CF8003CFFFFFCF80000F8 0000F80000F80000F800007800007C00003C00043C00041E00080E001007002001C0C000 7F00161A7E991B>I<001F000070C000E1E001C3E003C3E00381C0078080078000078000 078000078000078000078000078000078000078000FFFE00FFFE00078000078000078000 078000078000078000078000078000078000078000078000078000078000078000078000 07800007800007800007800007800007800007C000FFFE00FFFE00132A7FA912>I<0000 078001FC1840070721C00E03C1C01E03C0803C01E0003C01E0007C01F0007C01F0007C01 F0007C01F0007C01F0003C01E0003C01E0001E03C0000E0380001707000011FC00003000 00003000000030000000380000001C0000001FFFC0000FFFF80007FFFC001C003E003000 0F007000070060000380E0000380E0000380E0000380E000038070000700700007003800 0E000C0018000780F00000FF80001A287E9A1E>I<07800000FF800000FF8000000F8000 000780000007800000078000000780000007800000078000000780000007800000078000 000780000007800000078000000783F800078C1C0007900E0007A0070007A0078007C007 8007C0078007800780078007800780078007800780078007800780078007800780078007 800780078007800780078007800780078007800780078007800780078007800780078007 80FFFCFFFCFFFCFFFC1E2A7FA921>I<07000F801F801F800F8007000000000000000000 0000000000000000000007807F807F800F80078007800780078007800780078007800780 07800780078007800780078007800780078007800780FFF8FFF80D297FA811>I<078000 00FF800000FF8000000F8000000780000007800000078000000780000007800000078000 0007800000078000000780000007800000078000000780000007807FF007807FF007801F 8007801C0007801800078020000780400007808000078100000782000007870000079F80 0007A7800007C7C0000783E0000781E0000781F0000780F8000780780007807C0007803E 0007801E0007801F0007801F80FFFC7FF8FFFC7FF81D2A7FA920>107 D<0780FF80FF800F80078007800780078007800780078007800780078007800780078007 800780078007800780078007800780078007800780078007800780078007800780078007 800780078007800780FFFCFFFC0E2A7FA911>I<0781F800FC00FF860E030700FF98070C 03800FA0079003C007A003D001E007C003E001E007C003E001E0078003C001E0078003C0 01E0078003C001E0078003C001E0078003C001E0078003C001E0078003C001E0078003C0 01E0078003C001E0078003C001E0078003C001E0078003C001E0078003C001E0078003C0 01E0078003C001E0078003C001E0078003C001E0FFFC7FFE3FFFFFFC7FFE3FFF301A7F99 33>I<0783F800FF8C1C00FF900E000FA0070007A0078007C0078007C007800780078007 800780078007800780078007800780078007800780078007800780078007800780078007 800780078007800780078007800780078007800780078007800780FFFCFFFCFFFCFFFC1E 1A7F9921>I<007F000001C1C000070070000E0038001C001C003C001E003C001E007800 0F0078000F00F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F800 0F8078000F0078000F003C001E003C001E001E003C000E0038000700700001C1C000007F 0000191A7E991E>I<0783F000FF8C1C00FFB00F0007A0078007C003C0078003E0078001 E0078001F0078001F0078000F8078000F8078000F8078000F8078000F8078000F8078000 F8078000F0078001F0078001F0078001E0078003C007C003C007C0078007A00E0007983C 000787E00007800000078000000780000007800000078000000780000007800000078000 000780000007800000FFFC0000FFFC00001D267F9921>I<0787C0FF98E0FF91F00FA1F0 07C1F007C0E007C000078000078000078000078000078000078000078000078000078000 07800007800007800007800007800007800007800007C000FFFE00FFFE00141A7F9917> 114 D<07F8401C06C03001C06000C06000C0E00040E00040F00040F800007E00007FF000 3FFE000FFF0003FF80003FC00007C08001E08001E0C000E0C000E0C000E0E000C0F001C0 F80180C4070083F800131A7E9918>I<0080000080000080000080000180000180000180 000380000380000780000F80001FFF80FFFF800780000780000780000780000780000780 000780000780000780000780000780000780000780000780400780400780400780400780 4007804007804003C08001C08000E100003E0012257FA417>I<07800780FF80FF80FF80 FF800F800F80078007800780078007800780078007800780078007800780078007800780 078007800780078007800780078007800780078007800780078007800780078007800780 0F8007800F800380178001C027C000E047FC003F87FC1E1A7F9921>II121 D E /Fv 14 108 df0 D<70F8F8F87005057C8E0E>I<00C00000C00000C00000C00000C000C0C0C0F0C3C038C7 000EDC0003F00000C00003F0000EDC0038C700F0C3C0C0C0C000C00000C00000C00000C0 0000C00012157D9619>3 D<0000000C0000003C000000F0000003C000000F0000003C00 0000F0000007C000001F00000078000001E00000078000001E00000078000000E0000000 780000001E0000000780000001E0000000780000001F00000007C0000000F00000003C00 00000F00000003C0000000F00000003C0000000C00000000000000000000000000000000 000000000000000000000000000000007FFFFFF8FFFFFFFC1E277C9F27>20 DI29 D<0000000060000000000030000000000030000000000018000000000018000000 00000C00000000000600000000000380FFFFFFFFFFE0FFFFFFFFFFC00000000003800000 00000600000000000C000000000018000000000018000000000030000000000030000000 000060002B127D9432>33 D<03F80001F80007FE000FFE001E3F801C0300380FC0300180 2003E06000804001F0C000404000F9800040C0007F00002080003F00002080003E000020 80001F00002080000F80002080001F80002080001FC00060400033E00040400061F00040 2000C0F800803001807E03801807003F8F000FFE000FFC0003F00003F8002B157D9432> 49 D<001FFF007FFF01E0000380000600000C0000180000300000300000600000600000 600000C00000C00000FFFFFFFFFFFFC00000C00000600000600000600000300000300000 1800000C000006000003800001E000007FFF001FFF181E7C9A21>I<00001FFFFE0000FF FFFF0003FFFFFF000F1E000C00181E000800701E000000E01C000000E01C000001C03C00 000100380000000038000000007800000000780000000070000000007000000000F00000 0000FFFF000000FFFE000001FFFC000001C000000003C000000003800000000380000000 070000000007000000000F000000000E000000000E000000001C000000201C0000007038 000000F838000000FC700000007FC00000003F800000001E0000000028247FA124>70 D<000F0038007000E001C001C001C001C001C001C001C001C001C001C001C001C001C001 C001C001C001C0038007001E00F0001E000700038001C001C001C001C001C001C001C001 C001C001C001C001C001C001C001C001C001C000E000700038000F10317CA419>102 DI106 D<8010C030C030C030C030C030C030C030 C030C030C030C030C030C030C030C030C030C030C030C030C030C030C030C030C030C030 C030C030C030C030C030C030C030C030C030C030C030C030C030C030C030C030C030C030 C030C030C030C030C03080100C327AA419>I E /Fw 32 122 df<0000700000F00001C0 000380000780000F00001E00003C00007C0000F80000F80001F00003F00003E00007E000 07C0000FC0000FC0001F80001F80001F80003F00003F00003F00007F00007F00007E0000 7E00007E0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000 FE0000FE0000FE0000FE0000FE00007E00007E00007E00007F00007F00003F00003F0000 3F00001F80001F80001F80000FC0000FC00007C00007E00003E00003F00001F00000F800 00F800007C00003C00001E00000F000007800003800001C00000F000007014497AB520> 40 DI45 D<0001E0000003E000000FE000007FE0001FFFE000FFFFE000FFBFE000E03F E000003FE000003FE000003FE000003FE000003FE000003FE000003FE000003FE000003F E000003FE000003FE000003FE000003FE000003FE000003FE000003FE000003FE000003F E000003FE000003FE000003FE000003FE000003FE000003FE000003FE000003FE000003F E000003FE000003FE000003FE000003FE000003FE000003FE000003FE000003FE0007FFF FFF07FFFFFF07FFFFFF01C2E7AAD29>49 D<0000007C0000000000007C000000000000FE 000000000000FE000000000000FE000000000001FF000000000001FF000000000003FF80 0000000003FF800000000007FFC00000000007FFC00000000007FFC0000000000FFFE000 0000000F7FE0000000001F7FF0000000001E3FF0000000001E3FF0000000003E3FF80000 00003C1FF8000000007C1FFC00000000780FFC00000000780FFC00000000F80FFE000000 00F007FE00000001F007FF00000001E003FF00000001E003FF00000003E003FF80000003 C001FF80000007C001FFC00000078000FFC00000078000FFC000000FFFFFFFE000000FFF FFFFE000001FFFFFFFF000001E00003FF000001E00003FF000003C00003FF800003C0000 1FF800007C00001FFC00007800000FFC00007800000FFC0000F0000007FE0000F0000007 FE0001F0000007FF0003F8000003FF00FFFFC001FFFFFEFFFFC001FFFFFEFFFFC001FFFF FE37317DB03E>65 D<000003FF80018000003FFFF003800001FFFFFC0F800007FF007F1F 80001FF8000FBF80003FE00003FF8000FF800000FF8001FF0000007F8003FE0000003F80 07FC0000003F8007FC0000001F800FF80000001F801FF80000000F801FF00000000F803F F000000007803FF000000007807FF000000007807FE000000007807FE000000000007FE0 0000000000FFE00000000000FFE00000000000FFE00000000000FFE00000000000FFE000 00000000FFE00000000000FFE00000000000FFE00000000000FFE000000000007FE00000 0000007FE000000000007FE000000000007FF000000003803FF000000003803FF0000000 03801FF000000003801FF800000007800FF8000000070007FC000000070007FC0000000E 0003FE0000001E0001FF0000003C0000FF8000007800003FE00000F000001FF80003E000 0007FF003F80000001FFFFFE000000003FFFF80000000003FF80000031317BB03C>67 DI73 D<00000FFE0000000000FFFFE000000007FFFFFC0000 001FFC07FF0000003FE000FF800000FF80003FE00001FF00001FF00003FE00000FF80007 FC000007FC0007FC000007FC000FF8000003FE001FF8000003FF001FF0000001FF003FF0 000001FF803FF0000001FF803FF0000001FF807FE0000000FFC07FE0000000FFC07FE000 0000FFC0FFE0000000FFE0FFE0000000FFE0FFE0000000FFE0FFE0000000FFE0FFE00000 00FFE0FFE0000000FFE0FFE0000000FFE0FFE0000000FFE0FFE0000000FFE0FFE0000000 FFE0FFE0000000FFE07FE0000000FFC07FE0000000FFC07FF0000001FFC07FF0000001FF C03FF0000001FF803FF0000001FF801FF8000003FF001FF8000003FF000FFC000007FE00 0FFC000007FE0007FE00000FFC0003FF00001FF80001FF80003FF00000FFC0007FE00000 3FE000FF8000001FFC07FF00000007FFFFFC00000000FFFFE0000000000FFE0000003331 7BB03E>79 DI<00000FFE0000000000FFFFE000000007FF FFFC0000001FFC07FF0000003FE000FF800000FFC0007FE00001FF00001FF00003FE0000 0FF80007FE00000FFC0007FC000007FC000FF8000003FE001FF8000003FF001FF8000003 FF003FF0000001FF803FF0000001FF803FF0000001FF807FF0000001FFC07FE0000000FF C07FE0000000FFC0FFE0000000FFE0FFE0000000FFE0FFE0000000FFE0FFE0000000FFE0 FFE0000000FFE0FFE0000000FFE0FFE0000000FFE0FFE0000000FFE0FFE0000000FFE0FF E0000000FFE0FFE0000000FFE07FE0000000FFC07FE0000000FFC07FF0000001FFC07FF0 000001FFC03FF0000001FF803FF0000001FF801FF8000003FF001FF8000003FF000FF803 F003FE000FFC0FFC07FE0007FE1FFE0FFC0003FE1C0F0FF80001FF38079FF00000FFF803 FFE000003FF803FF8000001FFC07FF00000007FFFFFC00000000FFFFF0004000000FFEF8 00E000000000FC00E000000000FC01E000000000FF03E000000000FFFFE0000000007FFF E0000000007FFFC0000000007FFFC0000000003FFFC0000000003FFF80000000003FFF80 000000001FFF00000000000FFE000000000007FC000000000001F000333F7BB03E>II<001FF0018000FFFF038003FFFFC78007F00FFF800F8001FF801F00007F803F00001F 803E00000F807E00000F807E00000780FE00000780FE00000780FE00000380FF00000380 FF00000380FF80000000FFE00000007FFC0000007FFFE000007FFFFE00003FFFFFC0001F FFFFF0001FFFFFF8000FFFFFFC0003FFFFFE0001FFFFFF00007FFFFF80001FFFFF800000 FFFFC0000007FFC0000000FFE00000003FE00000003FE00000001FE06000001FE0E00000 0FE0E000000FE0E000000FE0E000000FC0F000000FC0F000000FC0F800001F80FC00001F 80FF00003F00FFC0007E00FFFC01FC00F1FFFFF800E03FFFE000C007FF000023317BB02E >I<3FFFFFFFFFFF003FFFFFFFFFFF003FFFFFFFFFFF003FE00FFC01FF007F000FFC003F 807E000FFC001F807C000FFC000F8078000FFC00078078000FFC00078070000FFC000380 70000FFC00038070000FFC00038070000FFC000380E0000FFC0001C0E0000FFC0001C0E0 000FFC0001C0E0000FFC0001C000000FFC00000000000FFC00000000000FFC0000000000 0FFC00000000000FFC00000000000FFC00000000000FFC00000000000FFC00000000000F FC00000000000FFC00000000000FFC00000000000FFC00000000000FFC00000000000FFC 00000000000FFC00000000000FFC00000000000FFC00000000000FFC00000000000FFC00 000000000FFC00000000000FFC00000000000FFC00000000000FFC00000000000FFC0000 0000000FFC00000000000FFC00000000000FFC00000000000FFC000000007FFFFFFF8000 007FFFFFFF8000007FFFFFFF800032307DAF39>I<007FF8000003FFFF000007FFFFC000 0FE01FE0001FF007F0001FF003F8001FF003FC001FF001FE000FE001FE0007C001FE0001 0001FE00000001FE00000001FE000001FFFE00003FFFFE0001FFF1FE0007FE01FE000FF0 01FE001FC001FE003F8001FE007F8001FE00FF0001FE00FF0001FE00FF0001FE00FF0001 FE00FF0003FE007F8003FE007FC00EFE003FF03CFF000FFFF87FF807FFF03FF800FF800F F825207E9F28>97 D<0007FF00007FFFE000FFFFF003FC03F807F007FC0FE007FC1FE007 FC3FC007FC3FC003F87FC001F07F8000407F800000FF800000FF800000FF800000FF8000 00FF800000FF800000FF800000FF8000007F8000007FC000007FC000003FC0000E3FE000 0E1FE0001C0FF0001C07F8007803FF01F000FFFFE0007FFF800007FC001F207D9F25>99 D<0007FC0000003FFF800000FFFFE00003FC07F00007F801F8000FE000FC001FE0007E00 3FC0007E003FC0003F007FC0003F007F80003F007F80003F80FF80003F80FF80003F80FF FFFFFF80FFFFFFFF80FFFFFFFF80FF80000000FF80000000FF800000007F800000007F80 0000003FC00000003FC00003801FC00003801FE00007800FF0000F0007F8001E0003FE00 FC0000FFFFF800003FFFE0000003FF000021207E9F26>101 D<0000FF000007FFC0001F FFE0003FC7F0007F0FF800FE0FF801FE0FF801FC0FF803FC07F003FC03E003FC01C003FC 000003FC000003FC000003FC000003FC000003FC000003FC0000FFFFF800FFFFF800FFFF F80003FC000003FC000003FC000003FC000003FC000003FC000003FC000003FC000003FC 000003FC000003FC000003FC000003FC000003FC000003FC000003FC000003FC000003FC 000003FC000003FC000003FC000003FC000003FC000003FC000003FC000003FC00007FFF F0007FFFF0007FFFF0001D327EB119>I<001FF007E000FFFE3FF001FFFF7FF807F83FF1 F80FE00FE1F80FE00FE0F01FC007F0601FC007F0003FC007F8003FC007F8003FC007F800 3FC007F8003FC007F8001FC007F0001FC007F0000FE00FE0000FE00FE00007F83FC00007 FFFF000006FFFE00000E1FF000000E000000001E000000001E000000001F000000001F80 0000001FFFFFC0000FFFFFF8000FFFFFFE0007FFFFFF0003FFFFFF8007FFFFFFC01FFFFF FFE03F00007FE07E00000FF0FC000007F0FC000003F0FC000003F0FC000003F0FC000003 F07E000007E03F00000FC01FC0003F800FF801FF0007FFFFFE0000FFFFF000001FFF8000 252F7E9F29>I<01F800000000FFF800000000FFF800000000FFF8000000000FF8000000 0007F80000000007F80000000007F80000000007F80000000007F80000000007F8000000 0007F80000000007F80000000007F80000000007F80000000007F80000000007F8000000 0007F80000000007F807F8000007F83FFF000007F87FFF800007F8F03FC00007F9C01FE0 0007FB000FE00007FE000FF00007FE000FF00007FC000FF00007FC000FF00007F8000FF0 0007F8000FF00007F8000FF00007F8000FF00007F8000FF00007F8000FF00007F8000FF0 0007F8000FF00007F8000FF00007F8000FF00007F8000FF00007F8000FF00007F8000FF0 0007F8000FF00007F8000FF00007F8000FF00007F8000FF00007F8000FF00007F8000FF0 00FFFFC1FFFF80FFFFC1FFFF80FFFFC1FFFF8029327DB12E>I<03C0000FF0000FF0001F F8001FF8001FFC001FF8001FF8000FF0000FF00003C00000000000000000000000000000 000000000000000000000001F800FFF800FFF800FFF8000FF80007F80007F80007F80007 F80007F80007F80007F80007F80007F80007F80007F80007F80007F80007F80007F80007 F80007F80007F80007F80007F80007F80007F80007F80007F800FFFF80FFFF80FFFF8011 337DB217>I<01F800FFF800FFF800FFF8000FF80007F80007F80007F80007F80007F800 07F80007F80007F80007F80007F80007F80007F80007F80007F80007F80007F80007F800 07F80007F80007F80007F80007F80007F80007F80007F80007F80007F80007F80007F800 07F80007F80007F80007F80007F80007F80007F80007F80007F80007F80007F80007F800 07F800FFFFC0FFFFC0FFFFC012327DB117>108 D<03F007F8000FF000FFF03FFF007FFE 00FFF07FFF80FFFF00FFF0F03FC1E07F800FF1C01FE3803FC007F3000FE6001FC007F600 0FFC001FE007FE000FFC001FE007FC000FF8001FE007FC000FF8001FE007F8000FF0001F E007F8000FF0001FE007F8000FF0001FE007F8000FF0001FE007F8000FF0001FE007F800 0FF0001FE007F8000FF0001FE007F8000FF0001FE007F8000FF0001FE007F8000FF0001F E007F8000FF0001FE007F8000FF0001FE007F8000FF0001FE007F8000FF0001FE007F800 0FF0001FE007F8000FF0001FE007F8000FF0001FE007F8000FF0001FE007F8000FF0001F E0FFFFC1FFFF83FFFFFFFFC1FFFF83FFFFFFFFC1FFFF83FFFF40207D9F45>I<03F007F8 0000FFF03FFF0000FFF07FFF8000FFF0F03FC0000FF1C01FE00007F3000FE00007F6000F F00007FE000FF00007FC000FF00007FC000FF00007F8000FF00007F8000FF00007F8000F F00007F8000FF00007F8000FF00007F8000FF00007F8000FF00007F8000FF00007F8000F F00007F8000FF00007F8000FF00007F8000FF00007F8000FF00007F8000FF00007F8000F F00007F8000FF00007F8000FF00007F8000FF00007F8000FF000FFFFC1FFFF80FFFFC1FF FF80FFFFC1FFFF8029207D9F2E>I<0007FE0000003FFFC00000FFFFF00003FC03FC0007 F000FE000FE0007F001FC0003F803FC0003FC03FC0003FC07F80001FE07F80001FE07F80 001FE0FF80001FF0FF80001FF0FF80001FF0FF80001FF0FF80001FF0FF80001FF0FF8000 1FF0FF80001FF07F80001FE07F80001FE07F80001FE03FC0003FC03FC0003FC01FE0007F 800FE0007F0007F801FE0003FE07FC0001FFFFF800003FFFC0000007FE000024207E9F29 >I<01F80FF000FFF87FFE00FFF9FFFF80FFFFE07FC00FFF001FE007FE000FF007F80007 F807F80007FC07F80003FC07F80003FE07F80003FE07F80001FE07F80001FF07F80001FF 07F80001FF07F80001FF07F80001FF07F80001FF07F80001FF07F80001FF07F80001FE07 F80003FE07F80003FE07F80003FC07F80007FC07FC0007F807FE000FF007FF001FE007FB E07FC007F9FFFF0007F87FFE0007F81FE00007F800000007F800000007F800000007F800 000007F800000007F800000007F800000007F800000007F800000007F800000007F80000 00FFFFC00000FFFFC00000FFFFC00000282E7E9F2E>I<03F03F00FFF07FC0FFF1FFE0FF F3C7F00FF38FF807F70FF807F60FF807FE0FF807FC07F007FC03E007FC008007F8000007 F8000007F8000007F8000007F8000007F8000007F8000007F8000007F8000007F8000007 F8000007F8000007F8000007F8000007F8000007F8000007F8000007F80000FFFFE000FF FFE000FFFFE0001D207E9F22>114 D<00FF870007FFEF001FFFFF003F007F003C001F00 78000F00F8000700F8000700F8000700FC000700FF000000FFF800007FFFC0003FFFF000 3FFFFC000FFFFE0007FFFF0001FFFF80001FFF800000FFC000001FC060000FC0E00007C0 E00007C0F00007C0F8000780F8000F80FE000F00FF803E00FFFFFC00F3FFF800C07FC000 1A207D9F21>I<00380000380000380000380000380000780000780000780000F80000F8 0001F80003F80007F8001FF800FFFFFEFFFFFEFFFFFE07F80007F80007F80007F80007F8 0007F80007F80007F80007F80007F80007F80007F80007F80007F80007F80007F80007F8 0707F80707F80707F80707F80707F80707F80703F80E03FC0E01FE1C00FFF8007FF0000F E0182E7EAD20>I<01F80003F000FFF801FFF000FFF801FFF000FFF801FFF0000FF8001F F00007F8000FF00007F8000FF00007F8000FF00007F8000FF00007F8000FF00007F8000F F00007F8000FF00007F8000FF00007F8000FF00007F8000FF00007F8000FF00007F8000F F00007F8000FF00007F8000FF00007F8000FF00007F8000FF00007F8000FF00007F8000F F00007F8000FF00007F8001FF00007F8001FF00003F8003FF00003F8006FF00001FE03CF F80000FFFF8FFF80007FFF0FFF80000FFC0FFF8029207D9F2E>II121 D E end %%EndProlog %%BeginSetup %%Feature: *Resolution 300dpi TeXDict begin %%EndSetup %%Page: 1 1 1 0 bop 195 426 a Fw(On)28 b(the)f(Con)n(v)n(ergence)e(of)j(Sto)r(c)n (hastic)e(Iterativ)n(e)324 549 y(Dynamic)h(Programming)h(Algorithms) 1599 523 y Fv(\003)759 730 y Fu(T)-5 b(ommi)17 b(Jaakk)n(ola)745 831 y(Mic)n(hael)h(I.)h(Jordan)393 932 y(Departmen)n(t)g(of)h(Brain)g (and)g(Cognitiv)n(e)f(Sciences)476 1045 y(Massac)n(h)n(usetts)g (Institute)g(of)h(T)-5 b(ec)n(hnology)746 1207 y(Satinder)20 b(P)-5 b(.)20 b(Singh)539 1320 y(Departmen)n(t)f(of)h(Computer)f (Science)452 1433 y(Univ)n(ersit)n(y)g(of)h(Massac)n(h)n(usetts)f(at)h (Amherst)p 150 1870 660 2 v 248 1918 a Ft(*This)c(pro)s(ject)f(w)o(as)h (supp)q(orted)g(in)f(part)h(b)o(y)f(a)h(gran)o(t)g(from)f(the)g (McDonnell-P)o(ew)150 2021 y(F)l(oundation,)21 b(b)o(y)f(a)h(gran)o(t)f (from)f(A)l(TR)h(Human)f(Information)g(Pro)q(cessing)i(Researc)o(h)150 2125 y(Lab)q(oratories,)c(b)o(y)e(a)h(gran)o(t)g(from)f(Siemens)e(Corp) q(oration,)k(and)f(b)o(y)f(gran)o(t)h(N00014-90-)150 2229 y(J-1942)23 b(from)e(the)g(O\016ce)g(of)h(Na)o(v)m(al)f(Researc)o (h.)36 b(The)22 b(pro)s(ject)f(w)o(as)h(also)g(supp)q(orted)150 2333 y(b)o(y)15 b(NSF)h(gran)o(t)g(ASC-9217041)i(in)e(supp)q(ort)h(of)f (the)f(Cen)o(ter)g(for)h(Biological)f(and)h(Com-)150 2437 y(putational)i(Learning)h(at)f(MIT,)f(including)g(funds)h(pro)o (vided)f(b)o(y)h(D)o(ARP)l(A)e(under)i(the)150 2540 y(HPCC)e(program.) 22 b(Mic)o(hael)14 b(I.)h(Jordan)j(is)e(a)g(NSF)g(Presiden)o(tial)f(Y)l (oung)i(In)o(v)o(estigator.)963 2775 y(1)p eop %%Page: 2 2 2 1 bop 875 192 a Fs(Abstract)340 310 y Fr(Recen)o(t)25 b(dev)o(elopmen)o(ts)g(in)h(the)e(area)g(of)g(reinforcemen)o(t)h (learning)g(ha)o(v)o(e)272 410 y(yielded)c(a)d(n)o(um)o(b)q(er)h(of)f (new)g(algorithms)h(for)f(the)g(prediction)j(and)d(con)o(trol)g(of)272 513 y(Mark)o(o)o(vian)c(en)o(vironmen)o(ts.)20 b(These)15 b(algorithms,)f(including)k(the)d(TD\()p Fq(\025)p Fr(\))e(algo-)272 618 y(rithm)f(of)g(Sutton)g(\(1988\))e(and)j(the)f(Q-learning)i (algorithm)e(of)g(W)l(atkins)g(\(1989\),)272 720 y(can)i(b)q(e)h(motiv) m(ated)g(heuristically)i(as)d(appro)o(ximations)g(to)g(dynamic)h (program-)272 823 y(ming)g(\(DP\).)e(In)i(this)g(pap)q(er)g(w)o(e)f (pro)o(vide)h(a)f(rigorous)g(pro)q(of)g(of)g(con)o(v)o(ergence)g(of)272 925 y(these)19 b(DP-based)g(learning)h(algorithms)f(b)o(y)g(relating)h (them)e(to)h(the)g(p)q(o)o(w)o(erful)272 1025 y(tec)o(hniques)i(of)e (sto)q(c)o(hastic)g(appro)o(ximation)g(theory)g(via)h(a)f(new)h(con)o (v)o(ergence)272 1125 y(theorem.)30 b(The)19 b(theorem)g(establishes)h (a)e(general)i(class)f(of)f(con)o(v)o(ergen)o(t)g(algo-)272 1227 y(rithms)d(to)g(whic)o(h)h(b)q(oth)f(TD\()p Fq(\025)p Fr(\))f(and)h(Q-learning)i(b)q(elong.)248 1382 y Ft(An)11 b(imp)q(ortan)o(t)g(comp)q(onen)o(t)g(of)h(man)o(y)e(real)h(w)o(orld)h (learning)g(problems)e(is)i(the)f Fp(tem-)150 1486 y(p)n(or)n(al)17 b(cr)n(e)n(dit)g(assignment)j(pr)n(oblem)p Ft(|the)c(problem)f(of)j (assigning)f(credit)f(or)h(blame)f(to)150 1590 y(individual)g(comp)q (onen)o(ts)g(of)h(a)h(temp)q(orally-extended)c(plan)j(of)h(action,)e (based)i(on)f(the)150 1694 y(success)f(or)g(failure)f(of)h(the)f(plan)h (as)g(a)g(whole.)21 b(T)l(o)c(solv)o(e)e(suc)o(h)g(a)h(problem,)e(the)i (learner)150 1798 y(m)o(ust)21 b(b)q(e)i(equipp)q(ed)f(with)g(the)g (abilit)o(y)f(to)i(assess)g(the)g(long-term)e(consequences)h(of)150 1901 y(particular)14 b(c)o(hoices)g(of)h(action)g(and)g(m)o(ust)e(b)q (e)i(willing)f(to)h(forego)h(an)f(imme)o(diate)d(pa)o(y)o(o\013)150 2005 y(for)21 b(the)g(prosp)q(ect)h(of)g(a)f(longer)h(term)d(gain.)37 b(Moreo)o(v)o(er,)20 b(b)q(ecause)i(most)e(real)h(w)o(orld)150 2109 y(problems)g(in)o(v)o(olving)g(prediction)g(of)i(the)f(future)g (consequences)f(of)i(actions)f(in)o(v)o(olv)o(e)150 2213 y(substan)o(tial)16 b(uncertain)o(t)o(y)l(,)d(the)j(learner)e(m)o(ust)g (b)q(e)i(prepared)f(to)h(mak)o(e)e(use)h(of)h(a)g(proba-)150 2316 y(bilit)o(y)e(calculus)i(for)g(assessing)i(and)f(comparing)e (actions.)248 2420 y(There)c(has)h(b)q(een)g(increasing)g(in)o(terest)e (in)h(the)h(temp)q(oral)f(credit)f(assignmen)o(t)h(prob-)150 2524 y(lem,)17 b(due)h(principally)f(to)i(the)g(dev)o(elopmen)o(t)c(of) 20 b(learning)e(algorithms)g(based)h(on)g(the)963 2775 y(2)p eop %%Page: 3 3 3 2 bop 150 192 a Ft(theory)15 b(of)g(dynamic)e(programming)g(\(DP\))i (\(Barto,)g(Sutton,)g(&)g(W)l(atkins,)g(1990;)h(W)l(er-)150 301 y(b)q(os,)d(1992\).)21 b(Sutton's)11 b(\(1988\))i(TD\()p Fo(\025)p Ft(\))e(algorithm)f(addressed)i(the)f(problem)e(of)i (learning)150 407 y(to)19 b(predict)e(in)h(a)g(Mark)o(o)o(v)g(en)o (vironmen)o(t,)d(utilizing)i(a)h(temp)q(oral)g(di\013erence)f(op)q (erator)150 514 y(to)23 b(up)q(date)g(the)f(predictions.)40 b(W)l(atkins')22 b(\(1989\))i(Q-learning)e(algorithm)g(extended)150 620 y(Sutton's)17 b(w)o(ork)f(to)g(con)o(trol)g(problems,)e(and)j(also) g(clari\014ed)e(the)h(ties)f(to)i(dynamic)d(pro-)150 723 y(gramming.)248 826 y(In)19 b(the)h(curren)o(t)g(pap)q(er,)h(our)f (concern)g(is)g(with)g(the)g(sto)q(c)o(hastic)g(con)o(v)o(ergence)f(of) 150 933 y(DP-based)k(learning)f(algorithms.)37 b(Although)22 b(W)l(atkins)g(\(1989\))i(and)e(W)l(atkins)g(and)150 1042 y(Da)o(y)o(an)j(\(1992\))h(pro)o(v)o(ed)f(that)g(Q-learning)g(con) o(v)o(erges)f(with)h(probabilit)o(y)f(one,)i(and)150 1152 y(Da)o(y)o(an)18 b(\(1992\))i(observ)o(ed)d(that)h(TD\(0\))h(is)f (a)g(sp)q(ecial)g(case)g(of)g(Q-learning)g(and)g(there-)150 1258 y(fore)h(also)h(con)o(v)o(erges)f(with)g(probabilit)o(y)f(one,)i (these)f(pro)q(ofs)i(rely)d(on)i(a)g(construction)150 1362 y(that)j(is)f(particular)g(to)g(Q-learning)g(and)h(fail)f(to)g (rev)o(eal)f(the)h(ties)g(of)g(Q-learning)h(to)150 1469 y(the)c(broad)h(theory)f(of)h(sto)q(c)o(hastic)f(appro)o(ximation)f (\(e.g.,)h(W)l(asan,)h(1969\).)32 b(Our)19 b(goal)150 1575 y(here)i(is)g(to)g(pro)o(vide)f(a)i(simpler)d(pro)q(of)j(of)g(con) o(v)o(ergence)d(for)i(Q-learning)h(b)o(y)e(making)150 1679 y(direct)f(use)h(of)g(sto)q(c)o(hastic)g(appro)o(ximation)f (theory)l(.)32 b(W)l(e)19 b(also)i(sho)o(w)f(that)h(our)f(pro)q(of)150 1785 y(extends)g(to)i(TD\()p Fo(\025)p Ft(\))f(for)g(arbitrary)g Fo(\025)p Ft(.)36 b(Sev)o(eral)19 b(other)i(authors)h(ha)o(v)o(e)e (recen)o(tly)f(pre-)150 1892 y(sen)o(ted)e(results)g(that)g(are)h (similar)d(to)i(those)h(presen)o(ted)e(here:)23 b(Da)o(y)o(an)18 b(and)f(Sejno)o(wski)150 1998 y(\(1993\))h(for)e(TD\()p Fo(\025)p Ft(\),)h(P)o(eng)f(and)h(Williams)d(\(1993\))k(for)f(TD\()p Fo(\025)p Ft(\),)f(and)h(Tsitsiklis)f(\(1993\))150 2108 y(for)h(Q-learning.)k(Our)16 b(results)g(app)q(ear)h(to)g(b)q(e)f (closest)g(to)h(those)f(of)h(Tsitsiklis)e(\(1993\).)248 2214 y(W)l(e)i(b)q(egin)h(with)f(a)h(general)g(o)o(v)o(erview)e(of)i (Mark)o(o)o(v)f(decision)f(problems)h(and)h(DP)l(.)150 2318 y(W)l(e)g(in)o(tro)q(duce)h(the)f(Q-learning)h(algorithm)f(as)h(a) g(sto)q(c)o(hastic)g(form)f(of)h(DP)l(.)g(W)l(e)f(then)150 2422 y(presen)o(t)j(a)g(pro)q(of)h(of)g(con)o(v)o(ergence)d(for)j(a)f (general)g(class)g(of)h(sto)q(c)o(hastic)f(pro)q(cesses)h(of)150 2528 y(whic)o(h)14 b(Q-learning)h(is)g(a)g(sp)q(ecial)f(case.)21 b(W)l(e)15 b(then)g(discuss)g(TD\()p Fo(\025)p Ft(\))g(and)h(sho)o(w)f (that)h(it)e(is)963 2775 y(3)p eop %%Page: 4 4 4 3 bop 150 192 a Ft(also)17 b(a)f(sp)q(ecial)g(case)g(of)h(our)g (theorem.)150 371 y Fn(Mark)n(o)n(v)25 b(decision)c(problems)150 509 y Ft(A)13 b(useful)g(mathematical)e(mo)q(del)h(of)i(temp)q(oral)f (credit)f(assignmen)o(t)h(problems,)f(studied)150 615 y(in)h(sto)q(c)o(hastic)g(con)o(trol)g(theory)g(\(Aoki,)g(1967\))h(and) g(op)q(erations)g(researc)o(h)f(\(Ross,)h(1970\),)150 722 y(is)20 b(the)g Fp(Markov)h(de)n(cision)g(pr)n(oblem)p Ft(.)33 b(Mark)o(o)o(v)19 b(decision)g(problems)g(are)h(built)g(on)g (the)150 826 y(formalism)12 b(of)i(con)o(trolled)f(Mark)o(o)o(v)h(c)o (hains.)20 b(Let)15 b Fo(S)h Ft(=)e(1)p Fo(;)8 b Ft(2)p Fo(;)g(:)g(:)g(:)g(;)g(N)20 b Ft(b)q(e)14 b(a)h(discrete)e(state)150 932 y(space)j(and)h(let)f Fo(U)5 b Ft(\()p Fo(i)p Ft(\))16 b(b)q(e)g(the)g(discrete)g(set)g(of)g(actions)h(a)o(v)m(ailable)f(to)g (the)g(learner)g(when)150 1039 y(the)k(c)o(hain)g(is)h(in)f(state)h Fo(i)p Ft(.)33 b(The)21 b(probabilit)o(y)e(of)i(making)e(a)i (transition)g(from)f(state)g Fo(i)150 1145 y Ft(to)e(state)g Fo(j)j Ft(is)d(giv)o(en)f(b)o(y)g Fo(p)646 1152 y Fm(ij)677 1145 y Ft(\()p Fo(u)p Ft(\),)g(where)h Fo(u)e Fv(2)h Fo(U)5 b Ft(\()p Fo(i)p Ft(\).)26 b(The)18 b(learner)f(de\014nes)h(a)g Fp(p)n(olicy)k Fo(\026)p Ft(,)150 1253 y(whic)o(h)14 b(is)h(a)h(function)f(from)f(states)h(to)h(actions.)21 b(Asso)q(ciated)15 b(with)g(ev)o(ery)f(p)q(olicy)g Fo(\026)i Ft(is)f(a)150 1360 y(Mark)o(o)o(v)g(c)o(hain)h(de\014ned)g(b)o(y)g(the) g(state)g(transition)h(probabilities)e Fo(p)1421 1367 y Fm(ij)1452 1360 y Ft(\()p Fo(\026)p Ft(\()p Fo(i)p Ft(\)\).)248 1471 y(There)23 b(is)h(an)g Fp(instantane)n(ous)i(c)n(ost) j Fo(c)974 1478 y Fm(i)988 1471 y Ft(\()p Fo(u)p Ft(\))23 b(asso)q(ciated)i(with)f(eac)o(h)g(state)g Fo(i)f Ft(and)150 1580 y(action)16 b Fo(u)p Ft(,)f(where)h Fo(c)515 1587 y Fm(i)529 1580 y Ft(\()p Fo(u)p Ft(\))f(is)h(a)g(random)f(v)m(ariable) h(with)g(exp)q(ected)e(v)m(alue)j(\026)-25 b Fo(c)1520 1587 y Fm(i)1534 1580 y Ft(\()p Fo(u)p Ft(\).)21 b(W)l(e)15 b(also)150 1689 y(de\014ne)h(a)i Fp(value)h(function)j Fo(V)682 1696 y Fm(\026)705 1689 y Ft(\()p Fo(i)p Ft(\),)16 b(whic)o(h)g(is)h(the)f(exp)q(ected)g(sum)g(of)h(discoun)o(ted)f (future)150 1797 y(costs)h(giv)o(en)e(that)i(the)f(system)f(b)q(egins)h (in)g(state)h Fo(i)f Ft(and)g(follo)o(ws)h(p)q(olicy)e Fo(\026)p Ft(:)551 1990 y Fo(V)579 1997 y Fm(\026)603 1990 y Ft(\()p Fo(i)p Ft(\))e(=)31 b(lim)723 2019 y Fm(N)t Fl(!1)834 1990 y Fo(E)s Fv(f)898 1936 y Fm(N)t Fl(\000)p Fk(1)906 1948 y Fj(X)907 2039 y Fm(t)p Fk(=0)982 1990 y Fo(\015)1010 1969 y Fm(t)1025 1990 y Fo(c)1046 1997 y Fm(s)1062 2001 y Fi(t)1079 1990 y Ft(\()p Fo(\026)p Ft(\()p Fo(s)1169 1997 y Fm(t)1184 1990 y Ft(\)\))p Fv(j)p Fo(s)1259 1997 y Fk(0)1292 1990 y Ft(=)14 b Fo(i)p Fv(g)p Fo(;)338 b Ft(\(1\))150 2180 y(where)21 b Fo(s)319 2187 y Fm(t)356 2180 y Fv(2)i Fo(S)h Ft(is)d(the)g(state)g(of)h(the)f(Mark)o (o)o(v)f(c)o(hain)h(at)g(time)f Fo(t)p Ft(.)35 b(F)l(uture)21 b(costs)h(are)150 2285 y(discoun)o(ted)e(b)o(y)h(a)g(factor)g Fo(\015)687 2267 y Fm(t)702 2285 y Ft(,)h(where)e Fo(\015)25 b Fv(2)d Ft(\(0)p Fo(;)8 b Ft(1\).)36 b(W)l(e)21 b(wish)g(to)g(\014nd)g (a)g(p)q(olicy)g(that)150 2392 y(minim)o(ize)o(s)14 b(the)i(v)m(alue)g (function:)781 2531 y Fo(V)820 2511 y Fl(\003)840 2531 y Ft(\()p Fo(i)p Ft(\))d(=)h(min)990 2556 y Fm(\026)1049 2531 y Fo(V)1077 2538 y Fm(\026)1101 2531 y Ft(\()p Fo(i)p Ft(\))p Fo(:)568 b Ft(\(2\))963 2775 y(4)p eop %%Page: 5 5 5 4 bop 150 192 a Ft(Suc)o(h)17 b(a)g(p)q(olicy)g(is)g(referred)f(to)i (as)g(an)f Fp(optimal)i(p)n(olicy)i Ft(and)d(the)f(corresp)q(onding)h (v)m(alue)150 295 y(function)h(is)h(referred)e(to)i(as)g(the)f Fp(optimal)i(value)h(function)p Ft(.)32 b(Note)19 b(that)h(the)f (optimal)150 402 y(v)m(alue)k(function)g(is)g(unique,)g(but)h(an)f (optimal)f(p)q(olicy)g(need)h(not)h(b)q(e)f(unique.)41 b(\(F)l(or)150 508 y(example,)13 b(if)j(the)f(costs)i(are)f(indep)q (enden)o(t)f(of)h(the)g(actions)g(and)g(states)h(all)e(p)q(olicies)g (are)150 615 y(optimal.\))248 724 y(Mark)o(o)o(v)e(decision)h(problems) f(can)i(b)q(e)g(solv)o(ed)f(b)o(y)g(dynamic)f(programming)g(\(Bert-)150 834 y(sek)m(as,)23 b(1987\).)37 b(The)21 b(basis)h(of)f(the)g(DP)h (approac)o(h)g(is)f(an)h(equation)f(that)g(c)o(haracter-)150 940 y(izes)15 b(the)h(optimal)f(v)m(alue)h(function.)21 b(This)16 b(equation,)g(kno)o(wn)g(as)h(Bellman's)d(equation,)150 1044 y(c)o(haracterizes)i(the)i(optimal)e(v)m(alue)h(of)h(the)f(state)h (in)g(terms)e(of)i(the)f(optimal)f(v)m(alues)i(of)150 1148 y(p)q(ossible)e(successor)h(states:)544 1300 y Fo(V)583 1280 y Fl(\003)603 1300 y Ft(\()p Fo(i)p Ft(\))c(=)29 b(min)723 1332 y Fm(u)p Fl(2)p Fm(U)t Fk(\()p Fm(i)p Fk(\))834 1300 y Fv(f)q Ft(\026)-25 b Fo(c)880 1307 y Fm(i)894 1300 y Ft(\()p Fo(u)p Ft(\))11 b(+)g Fo(\015)1057 1259 y Fj(X)1056 1351 y Fm(j)r Fl(2)p Fm(S)1127 1300 y Fo(p)1151 1307 y Fm(ij)1182 1300 y Ft(\()p Fo(u)p Ft(\))p Fo(V)1287 1280 y Fl(\003)1307 1300 y Ft(\()p Fo(j)s Ft(\))p Fv(g)p Fo(:)331 b Ft(\(3\))150 1493 y(T)l(o)14 b(motiv)m(ate)e(Bellm)o (an's)f(equation,)i(supp)q(ose)i(that)e(the)g(system)f(is)h(in)g(state) g Fo(i)g Ft(at)g(time)e Fo(t)150 1599 y Ft(and)i(consider)f(ho)o(w)h Fo(V)564 1581 y Fl(\003)584 1599 y Ft(\()p Fo(i)p Ft(\))f(should)h(b)q (e)g(c)o(haracterized)e(in)h(terms)f(of)i(p)q(ossible)g(transitions)150 1706 y(out)j(of)g(state)g Fo(i)p Ft(.)21 b(Supp)q(ose)16 b(that)h(action)e Fo(u)h Ft(is)f(selected)g(and)h(the)f(system)f (transitions)j(to)150 1812 y(state)h Fo(j)s Ft(.)24 b(The)18 b(expression)f Fo(c)691 1819 y Fm(i)705 1812 y Ft(\()p Fo(u)p Ft(\))11 b(+)h Fo(\015)s(V)899 1794 y Fl(\003)919 1812 y Ft(\()p Fo(j)s Ft(\))17 b(is)h(the)f(cost)g(of)h(making)e(a)i (transition)g(out)150 1919 y(of)i(state)g Fo(i)f Ft(plus)g(the)g (discoun)o(ted)g(cost)h(of)g(follo)o(wing)f(an)h(optimal)e(p)q(olicy)h (thereafter.)150 2023 y(The)14 b(minim)n(um)c(of)k(the)g(exp)q(ected)e (v)m(alue)i(of)g(this)g(expression,)g(o)o(v)o(er)f(p)q(ossible)h(c)o (hoices)f(of)150 2127 y(actions,)18 b(seems)e(a)h(plausible)g(measure)f (of)i(the)f(optimal)f(cost)i(at)g Fo(i)f Ft(and)h(b)o(y)f(Bellman's)150 2233 y(equation)f(is)g(indeed)g(equal)f(to)i Fo(V)780 2215 y Fl(\003)800 2233 y Ft(\()p Fo(i)p Ft(\).)248 2340 y(There)c(are)g(a)h(v)m(ariet)o(y)f(of)h(computational)e(tec)o(hniques) g(a)o(v)m(ailable)h(for)h(solving)f(Bell-)150 2443 y(man's)21 b(equation.)37 b(The)21 b(tec)o(hnique)f(that)i(w)o(e)f(fo)q(cus)i(on)f (in)f(the)g(curren)o(t)g(pap)q(er)h(is)g(a)150 2550 y(iterativ)o(e)16 b(algorithm)h(kno)o(wn)h(as)h Fp(value)h(iter)n(ation)p Ft(.)26 b(V)l(alue)17 b(iteration)h(solv)o(es)f(for)h Fo(V)1726 2532 y Fl(\003)1745 2550 y Ft(\()p Fo(i)p Ft(\))963 2775 y(5)p eop %%Page: 6 6 6 5 bop 150 192 a Ft(b)o(y)21 b(setting)h(up)g(a)g(recurrence)f (relation)g(for)h(whic)o(h)g(Bellman's)d(equation)i(is)h(a)g(\014xed) 150 304 y(p)q(oin)o(t.)f(Denoting)16 b(the)g(estimate)e(of)i Fo(V)879 286 y Fl(\003)899 304 y Ft(\()p Fo(i)p Ft(\))f(at)h(the)g Fo(k)1139 286 y Fm(th)1190 304 y Ft(iteration)f(as)i Fo(V)1486 286 y Fk(\()p Fm(k)q Fk(\))1535 304 y Ft(\()p Fo(i)p Ft(\),)e(w)o(e)g(ha)o(v)o(e:)499 453 y Fo(V)538 432 y Fk(\()p Fm(k)q Fk(+1\))632 453 y Ft(\()p Fo(i)p Ft(\))f(=)28 b(min)752 484 y Fm(u)p Fl(2)p Fm(U)t Fk(\()p Fm(i)p Fk(\))863 453 y Fv(f)q Ft(\026)-25 b Fo(c)909 460 y Fm(i)923 453 y Ft(\()p Fo(u)p Ft(\))11 b(+)g Fo(\015)1087 411 y Fj(X)1085 503 y Fm(j)r Fl(2)p Fm(S)1157 453 y Fo(p)1181 460 y Fm(ij)1211 453 y Ft(\()p Fo(u)p Ft(\))p Fo(V)1316 432 y Fk(\()p Fm(k)q Fk(\))1365 453 y Ft(\()p Fo(j)s Ft(\))p Fv(g)287 b Ft(\(4\))150 647 y(This)18 b(iteration)g(can)h(b)q (e)f(sho)o(wn)h(to)g(con)o(v)o(erge)e(to)i Fo(V)1137 629 y Fl(\003)1157 647 y Ft(\()p Fo(i)p Ft(\))f(for)h(arbitrary)f (initial)f Fo(V)1698 629 y Fk(\(0\))1745 647 y Ft(\()p Fo(i)p Ft(\))150 757 y(\(Bertsek)m(as,)24 b(1987\).)45 b(The)23 b(pro)q(of)i(is)e(based)h(on)g(sho)o(wing)g(that)g(the)f (iteration)g(from)150 872 y Fo(V)189 853 y Fk(\()p Fm(k)q Fk(\))238 872 y Ft(\()p Fo(i)p Ft(\))16 b(to)g Fo(V)408 853 y Fk(\()p Fm(k)q Fk(+1\))502 872 y Ft(\()p Fo(i)p Ft(\))f(is)i(a)f(con)o(traction)g(mapping.)21 b(That)c(is,)e(it)h(can)g (b)q(e)h(sho)o(wn)g(that:)455 1020 y(max)494 1049 y Fm(i)554 1020 y Fv(j)p Fo(V)607 1000 y Fk(\()p Fm(k)q Fk(+1\))701 1020 y Ft(\()p Fo(i)p Ft(\))10 b Fv(\000)h Fo(V)855 1000 y Fl(\003)875 1020 y Ft(\()p Fo(i)p Ft(\))p Fv(j)i(\024)h Fo(\015)d Ft(max)1086 1049 y Fm(i)1145 1020 y Fv(j)p Fo(V)1198 1000 y Fk(\()p Fm(k)q Fk(\))1247 1020 y Ft(\()p Fo(i)p Ft(\))g Fv(\000)g Fo(V)1402 1000 y Fl(\003)1422 1020 y Ft(\()p Fo(i)p Ft(\))p Fv(j)p Fo(;)233 b Ft(\(5\))150 1187 y(whic)o(h)15 b(implies)e(that)j Fo(V)599 1169 y Fk(\()p Fm(k)q Fk(\))647 1187 y Ft(\()p Fo(i)p Ft(\))g(con)o(v)o(erges) f(to)h Fo(V)1034 1169 y Fl(\003)1054 1187 y Ft(\()p Fo(i)p Ft(\))f(and)i(also)f(places)f(an)i(upp)q(er)f(b)q(ound)150 1293 y(on)h(the)f(con)o(v)o(ergence)e(rate.)248 1400 y(W)l(atkins)j(\(1989\))i(utilized)d(an)j(alternativ)o(e)d(notation)i (for)g(expressing)g(Bellman's)150 1507 y(equation)f(that)g(is)f (particularly)g(con)o(v)o(enien)o(t)f(for)i(deriving)e(learning)i (algorithms.)22 b(De-)150 1613 y(\014ne)h(the)h(function)f Fo(Q)578 1595 y Fl(\003)597 1613 y Ft(\()p Fo(i;)8 b(u)p Ft(\))23 b(to)h(b)q(e)g(the)f(expression)g(app)q(earing)i(inside)e(the) g(\\min)o(")150 1720 y(op)q(erator)17 b(of)g(Bellman's)c(equation:)607 1866 y Fo(Q)646 1845 y Fl(\003)665 1866 y Ft(\()p Fo(i;)8 b(u)p Ft(\))13 b(=)i(\026)-25 b Fo(c)856 1873 y Fm(i)870 1866 y Ft(\()p Fo(u)p Ft(\))10 b(+)i Fo(\015)1033 1824 y Fj(X)1032 1916 y Fm(j)r Fl(2)p Fm(S)1103 1866 y Fo(p)1127 1873 y Fm(ij)1158 1866 y Ft(\()p Fo(u)p Ft(\))p Fo(V)1263 1845 y Fl(\003)1283 1866 y Ft(\()p Fo(j)s Ft(\))394 b(\(6\))150 2052 y(Using)16 b(this)g(notation)h(Bellman's)d(equation)i(can)g(b)q(e) h(written)e(as)i(follo)o(ws:)738 2190 y Fo(V)777 2169 y Fl(\003)797 2190 y Ft(\()p Fo(i)p Ft(\))c(=)29 b(min)917 2222 y Fm(u)p Fl(2)p Fm(U)t Fk(\()p Fm(i)p Fk(\))1036 2190 y Fo(Q)1075 2169 y Fl(\003)1094 2190 y Ft(\()p Fo(i;)8 b(u)p Ft(\))p Fo(:)525 b Ft(\(7\))150 2360 y(Moreo)o(v)o(er,)15 b(v)m(alue)h(iteration)f(can)i(b)q(e)f(expressed)g(in)g(terms)f(of)h(Q) g(functions:)544 2506 y Fo(Q)583 2485 y Fk(\()p Fm(k)q Fk(+1\))676 2506 y Ft(\()p Fo(i;)8 b(u)p Ft(\))13 b(=)i(\026)-25 b Fo(c)867 2513 y Fm(i)881 2506 y Ft(\()p Fo(u)p Ft(\))11 b(+)g Fo(\015)1045 2464 y Fj(X)1043 2556 y Fm(j)r Fl(2)p Fm(S)1115 2506 y Fo(p)1139 2513 y Fm(ij)1170 2506 y Ft(\()p Fo(u)p Ft(\))p Fo(V)1274 2485 y Fk(\()p Fm(k)q Fk(\))1323 2506 y Ft(\()p Fo(j)s Ft(\))p Fo(;)340 b Ft(\(8\))963 2775 y(6)p eop %%Page: 7 7 7 6 bop 150 193 a Ft(where)16 b Fo(V)330 175 y Fk(\()p Fm(k)q Fk(\))379 193 y Ft(\()p Fo(i)p Ft(\))g(is)g(de\014ned)g(in)g (terms)e(of)j Fo(Q)954 175 y Fk(\()p Fm(k)q Fk(\))1002 193 y Ft(\()p Fo(i;)8 b(u)p Ft(\))16 b(as)h(follo)o(ws:)709 360 y Fo(V)748 340 y Fk(\()p Fm(k)q Fk(\))797 360 y Ft(\()p Fo(i)p Ft(\))c(=)29 b(min)917 392 y Fm(u)p Fl(2)p Fm(U)t Fk(\()p Fm(i)p Fk(\))1036 360 y Fo(Q)1075 340 y Fk(\()p Fm(k)q Fk(\))1124 360 y Ft(\()p Fo(i;)8 b(u)p Ft(\))p Fo(:)495 b Ft(\(9\))150 548 y(The)11 b(mathematical)e(con)o(v)o (enience)g(obtained)i(from)f(using)i Fo(Q)p Ft('s)f(rather)g(than)h Fo(V)f Ft('s)g(deriv)o(es)150 652 y(from)18 b(the)g(fact)g(that)h(the)g (minim)o(izati)o(on)e(op)q(erator)i(app)q(ears)h(inside)e(the)g(exp)q (ectation)150 756 y(in)f(Equation)g(8,)g(whereas)h(it)e(app)q(ears)j (outside)e(the)f(the)h(exp)q(ectation)g(in)g(Equation)g(4.)150 860 y(This)i(fact)g(pla)o(ys)g(an)g(imp)q(ortan)o(t)f(role)h(in)f(the)h (con)o(v)o(ergence)e(pro)q(of)j(presen)o(ted)f(in)f(this)150 950 y(pap)q(er.)248 1054 y(The)i(v)m(alue)g(iteration)g(algorithm)f(in) h(Equation)h(4)g(or)g(Equation)g(8)f(can)h(also)g(b)q(e)150 1161 y(executed)d Fp(asynchr)n(onously)23 b Ft(\(Bertsek)m(as)18 b(&)h(Tsitsiklis,)g(1989\).)31 b(In)18 b(an)i(async)o(hronous)150 1267 y(implem)o(en)o(tati)o(on,)13 b(the)i(up)q(date)i(of)e(the)h(v)m (alue)f(of)h(a)g(particular)f(state)h(ma)o(y)e(pro)q(ceed)h(in-)150 1371 y(dep)q(enden)o(tly)d(of)h(the)g(up)q(dates)h(of)f(the)g(v)m (alues)g(of)g(other)g(states.)21 b(Bertsek)m(as)12 b(&)h(Tsitsiklis)150 1478 y(\(1989\))j(sho)o(w)g(that)f(as)h(long)f(as)h(eac)o(h)f(state)g (is)g(up)q(dated)g(in\014nitely)f(often)h(and)g(eac)o(h)g(ac-)150 1584 y(tion)i(is)f(tried)h(an)g(in\014nite)f(n)o(um)o(b)q(er)f(of)i (times)e(in)i(eac)o(h)f(state,)h(then)f(the)h(async)o(hronous)150 1688 y(algorithm)12 b(ev)o(en)o(tually)g(con)o(v)o(erges)g(to)i(the)f (optimal)g(v)m(alue)g(function.)20 b(Moreo)o(v)o(er,)12 b(asyn-)150 1792 y(c)o(hronous)i(execution)e(has)i(the)g(adv)m(an)o (tage)g(that)g(it)f(is)g(directly)f(applicable)g(to)i Fp(r)n(e)n(al-time)150 1898 y Ft(Mark)o(o)o(v)g(decision)h(problems)f (\(R)l(TDP;)h(Barto,)h(Bradtk)o(e,)e(&)h(Singh,)g(1993\).)23 b(In)14 b(a)i(real-)150 2005 y(time)21 b(setting,)j(the)f(system)f (uses)h(its)g(ev)o(olving)f(v)m(alue)h(function)g(to)h(c)o(ho)q(ose)f (con)o(trol)150 2109 y(actions)d(for)g(an)g(actual)f(pro)q(cess)h(and)h (up)q(dates)f(the)f(v)m(alues)h(of)g(the)f(states)h(along)g(the)150 2212 y(tra)s(jectory)c(follo)o(w)o(ed)f(b)o(y)h(the)g(pro)q(cess.)248 2316 y(Dynamic)j(programming)g(serv)o(es)h(as)h(a)h(starting)f(p)q(oin) o(t)g(for)g(deriving)f(a)h(v)m(ariet)o(y)150 2420 y(of)h(learning)g (algorithms)g(for)g(systems)f(that)h(in)o(teract)f(with)h(Mark)o(o)o(v) g(en)o(vironmen)o(ts)150 2526 y(\(Barto,)c(Bradtk)o(e,)f(&)h(Singh,)g (1993;)i(Sutton,)f(1988;)h(W)l(atkins,)e(1989\).)28 b(Indeed,)17 b(real-)963 2775 y(7)p eop %%Page: 8 8 8 7 bop 150 192 a Ft(time)21 b(dynamic)h(programming)g(is)h(arguably)h (a)f(form)g(of)g(learning)g(algorithm)g(as)g(it)150 295 y(stands.)30 b(Although)19 b(R)l(TDP)h(requires)e(that)h(the)g(system)e (p)q(ossess)j(a)g(complete)c(mo)q(del)150 402 y(of)i(the)f(en)o (vironmen)o(t)e(\(i.e.,)h(the)i(probabilities)e Fo(p)1077 409 y Fm(ij)1108 402 y Ft(\()p Fo(u)p Ft(\))i(and)g(the)f(exp)q(ected)g (costs)i(\026)-25 b Fo(c)1720 409 y Fm(i)1734 402 y Ft(\()p Fo(u)p Ft(\))150 513 y(are)15 b(assumed)g(kno)o(wn\),)h(the)f(p)q (erformance)f(of)i(a)f(system)f(using)i(R)l(TDP)g(impro)o(v)o(es)d(o)o (v)o(er)150 619 y(time,)k(and)i(its)g(impro)o(v)o(em)o(e)o(n)o(t)d(is)i (fo)q(cused)i(on)f(the)f(states)i(that)f(are)g(actually)f(visited.)150 726 y(The)j(system)f(\\learns")i(b)o(y)f(transforming)g(kno)o(wledge)f (in)h(one)h(format)e(\(the)h(mo)q(del\))150 835 y(in)o(to)16 b(another)h(format)e(\(the)h(v)m(alue)g(function\).)248 942 y(A)g(more)g(di\016cult)f(learning)i(problem)e(arises)i(when)g(the) f(probabilistic)g(structure)150 1045 y(of)e(the)f(en)o(vironmen)o(t)e (is)j(unkno)o(wn.)20 b(There)14 b(are)f(t)o(w)o(o)h(approac)o(hes)g(to) g(dealing)g(with)f(this)150 1152 y(situation)k(\(cf.)22 b(Barto,)17 b(Bradtk)o(e,)f(&)g(Singh,)h(1993\).)24 b(An)17 b Fp(indir)n(e)n(ct)k Ft(approac)o(h)d(acquires)150 1258 y(a)k(mo)q(del)e(of)i(the)f(en)o(vironmen)o(t)e(incremen)o(tall)o(y)l (,)g(b)o(y)i(estimating)f(the)h(costs)h(and)g(the)150 1362 y(transition)e(probabilities,)e(and)i(then)g(uses)f(this)g(mo)q (del)f(in)i(an)f(ongoing)i(DP)f(compu-)150 1466 y(tation.)33 b(A)19 b Fp(dir)n(e)n(ct)25 b Ft(metho)q(d)19 b(disp)q(enses)h(with)g (constructing)g(a)h(mo)q(del)d(and)j(attempts)150 1573 y(to)d(estimate)e(the)h(optimal)f(v)m(alue)i(function)f(\(or)h(the)g (optimal)e(Q-v)m(alues\))i(directly)l(.)23 b(In)150 1679 y(the)f(remainder)e(of)i(this)g(pap)q(er,)i(w)o(e)d(fo)q(cus)i(on)f (direct)f(metho)q(ds,)i(in)f(particular)f(the)150 1786 y(Q-learning)e(algorithm)g(of)h(W)l(atkins)f(\(1989\))i(and)f(the)g (TD\()p Fo(\025)p Ft(\))g(algorithm)f(of)g(Sutton)150 1895 y(\(1988\).)248 2001 y(The)13 b(Q-learning)g(algorithm)f(is)i(a)f (sto)q(c)o(hastic)h(form)e(of)i(v)m(alue)f(iteration.)20 b(Consider)150 2105 y(Equation)12 b(8,)h(whic)o(h)e(expresses)g(the)g (up)q(date)i(of)f(the)f(Q)h(v)m(alues)f(in)h(terms)e(of)i(the)f(Q)h(v)m (alues)150 2209 y(of)23 b(successor)h(states.)42 b(T)l(o)23 b(p)q(erform)g(a)g(step)g(of)g(v)m(alue)g(iteration)g(requires)f(kno)o (wing)150 2313 y(the)i(exp)q(ected)e(costs)j(and)f(the)f(transition)h (probabilities.)43 b(Although)24 b(suc)o(h)g(a)g(step)150 2417 y(cannot)12 b(b)q(e)g(p)q(erformed)f(without)h(a)g(mo)q(del,)f(it) g(is)h(nonetheless)f(p)q(ossible)h(to)g Fp(estimate)k Ft(the)150 2523 y(appropriate)d(up)q(date.)20 b(F)l(or)13 b(an)g(arbitrary)f(V)g(function,)g(the)g(quan)o(tit)o(y)1462 2490 y Fj(P)1506 2534 y Fm(j)r Fl(2)p Fm(S)1579 2523 y Fo(p)1603 2530 y Fm(ij)1634 2523 y Ft(\()p Fo(u)p Ft(\))p Fo(V)f Ft(\()p Fo(j)s Ft(\))963 2775 y(8)p eop %%Page: 9 9 9 8 bop 150 192 a Ft(can)23 b(b)q(e)f(estimated)f(b)o(y)h(the)g(quan)o (tit)o(y)f Fo(V)12 b Ft(\()p Fo(j)s Ft(\),)23 b(if)f(successor)h(state) f Fo(j)k Ft(is)c(c)o(hosen)g(with)150 301 y(probabilit)o(y)c Fo(p)426 308 y Fm(ij)457 301 y Ft(\()p Fo(u)p Ft(\).)29 b(But)18 b(this)h(is)g(assured)h(b)o(y)e(simply)f(follo)o(wing)i(the)f (transitions)i(of)150 409 y(the)13 b(actual)h(Mark)o(o)o(v)f(en)o (vironmen)o(t,)e(whic)o(h)i(mak)o(es)e(a)j(transition)g(from)f(state)h Fo(i)f Ft(to)h(state)150 515 y Fo(j)21 b Ft(with)d(probabilit)o(y)g Fo(p)580 522 y Fm(ij)611 515 y Ft(\()p Fo(u)p Ft(\).)27 b(Th)o(us)18 b(the)g(sample)f(v)m(alue)h(of)h Fo(V)30 b Ft(at)18 b(the)g(successor)h(state)150 626 y(is)g(an)h(un)o(biased)g (estimate)d(of)j(the)f(sum.)30 b(Moreo)o(v)o(er)18 b Fo(c)1193 633 y Fm(i)1207 626 y Ft(\()p Fo(u)p Ft(\))h(is)h(an)g(un)o (biased)f(estimate)150 735 y(of)e(\026)-25 b Fo(c)226 742 y Fm(i)240 735 y Ft(\()p Fo(u)p Ft(\).)21 b(This)16 b(reasoning)h(leads)f(to)g(the)g(follo)o(wing)g(relaxation)f (algorithm,)g(where)g(w)o(e)150 845 y(use)i Fo(Q)274 852 y Fm(t)288 845 y Ft(\()p Fo(i;)8 b(u)p Ft(\))17 b(and)g Fo(V)533 852 y Fm(t)548 845 y Ft(\()p Fo(i)p Ft(\))g(to)g(denote)g(the) g(learner's)f(estimates)f(of)i(the)g(Q)g(function)f(and)150 951 y(V)g(function)g(at)g(time)f Fo(t)p Ft(,)g(resp)q(ectiv)o(ely:)199 1101 y Fo(Q)238 1108 y Fm(t)p Fk(+1)297 1101 y Ft(\()p Fo(s)339 1108 y Fm(t)354 1101 y Fo(;)8 b(u)404 1108 y Fm(t)418 1101 y Ft(\))14 b(=)g(\(1)d Fv(\000)g Fo(\013)638 1108 y Fm(t)653 1101 y Ft(\()p Fo(s)695 1108 y Fm(t)710 1101 y Fo(;)d(u)760 1108 y Fm(t)774 1101 y Ft(\)\))p Fo(Q)851 1108 y Fm(t)865 1101 y Ft(\()p Fo(s)907 1108 y Fm(t)922 1101 y Fo(;)g(u)972 1108 y Fm(t)987 1101 y Ft(\))j(+)g Fo(\013)1097 1108 y Fm(t)1112 1101 y Ft(\()p Fo(s)1154 1108 y Fm(t)1168 1101 y Fo(;)d(u)1218 1108 y Fm(t)1233 1101 y Ft(\)[)p Fo(c)1287 1108 y Fm(s)1303 1112 y Fi(t)1319 1101 y Ft(\()p Fo(u)1366 1108 y Fm(t)1380 1101 y Ft(\))j(+)g Fo(\015)s(V)1515 1108 y Fm(t)1530 1101 y Ft(\()p Fo(s)1572 1108 y Fm(t)p Fk(+1)1632 1101 y Ft(\)])48 b(\(10\))150 1250 y(where)669 1347 y Fo(V)697 1354 y Fm(t)712 1347 y Ft(\()p Fo(s)754 1354 y Fm(t)p Fk(+1)814 1347 y Ft(\))14 b(=)56 b(min)898 1379 y Fm(u)p Fl(2)p Fm(U)t Fk(\()p Fm(s)1000 1384 y Fi(t)p Fh(+1)1052 1379 y Fk(\))1074 1347 y Fo(Q)1113 1354 y Fm(t)1128 1347 y Ft(\()p Fo(s)1170 1354 y Fm(t)1184 1347 y Fo(;)8 b(u)1234 1354 y Fm(t)1249 1347 y Ft(\))p Fo(:)431 b Ft(\(11\))150 1512 y(The)17 b(v)m(ariables)g Fo(\013)485 1519 y Fm(t)500 1512 y Ft(\()p Fo(s)542 1519 y Fm(t)556 1512 y Fo(;)8 b(u)606 1519 y Fm(t)621 1512 y Ft(\))17 b(are)g(zero)g(except)e(for)j (the)e(state)i(that)f(is)g(b)q(eing)g(up)q(dated)h(at)150 1617 y(time)c Fo(t)p Ft(.)248 1711 y(The)k(fact)h(that)g(Q-learning)f (is)h(a)g(sto)q(c)o(hastic)g(form)e(of)i(v)m(alue)f(iteration)h(imm)o (edi-)150 1815 y(ately)i(suggests)h(the)f(use)h(of)f(sto)q(c)o(hastic)h (appro)o(ximation)e(theory)l(,)i(in)f(particular)g(the)150 1922 y(classical)i(framew)o(ork)f(of)h(Robbins)h(and)g(Monro)g (\(1951\).)45 b(Robbins-Monro)24 b(theory)150 2028 y(treats)17 b(the)g(sto)q(c)o(hastic)g(con)o(v)o(ergence)e(of)j(a)f(sequence)f(of)h (un)o(biased)g(estimates)e(of)j(a)f(re-)150 2132 y(gression)e (function,)f(pro)o(viding)g(conditions)h(under)f(whic)o(h)g(the)h (sequence)e(con)o(v)o(erges)h(to)150 2236 y(a)e(ro)q(ot)g(of)g(the)f (function.)19 b(Although)12 b(the)f(sto)q(c)o(hastic)h(con)o(v)o (ergence)d(of)j(Q-learning)f(is)h(not)150 2340 y(an)17 b(imme)o(diate)c(consequence)i(of)i(Robbins-Monro)g(theory)l(,)f(the)g (theory)g(do)q(es)h(pro)o(vide)150 2443 y(results)e(that)g(can)g(b)q(e) g(adapted)h(to)f(studying)g(the)g(con)o(v)o(ergence)e(of)i(DP-based)h (learning)150 2550 y(algorithms.)27 b(In)18 b(this)g(pap)q(er)h(w)o(e)e (utilize)g(a)h(result)g(from)f(Dv)o(oretzky's)g(\(1956\))j(form)o(u-) 963 2775 y(9)p eop %%Page: 10 10 10 9 bop 150 192 a Ft(lation)18 b(of)g(Robbins-Monro)h(theory)f(to)g (pro)o(v)o(e)f(the)h(con)o(v)o(ergence)e(of)i(b)q(oth)h(Q-learning)150 298 y(and)e(TD\()p Fo(\025)p Ft(\).)150 492 y Fw(Con)n(v)n(ergence)25 b(pro)r(of)k(for)e(Q-learning)150 649 y Ft(Our)19 b(pro)q(of)h(is)f (based)h(on)f(the)g(observ)m(ation)h(that)g(the)f(Q-learning)g (algorithm)f(can)h(b)q(e)150 753 y(view)o(ed)f(as)j(a)f(sto)q(c)o (hastic)g(pro)q(cess)g(to)g(whic)o(h)f(tec)o(hniques)f(of)i(sto)q(c)o (hastic)g(appro)o(xima-)150 857 y(tion)f(are)h(generally)e(applicable.) 29 b(Due)19 b(to)h(the)f(lac)o(k)f(of)i(a)f(form)o(ulation)f(of)i(sto)q (c)o(hastic)150 960 y(appro)o(ximation)f(for)h(the)g(maxim)n(um)c (norm,)k(ho)o(w)o(ev)o(er,)f(w)o(e)g(need)h(to)g(sligh)o(tly)f(extend) 150 1064 y(the)c(standard)h(results.)21 b(This)15 b(is)g(accomplished)e (b)o(y)h(the)h(follo)o(wing)g(theorem)e(the)i(pro)q(of)150 1168 y(of)i(whic)o(h)e(is)h(giv)o(en)g(in)f(App)q(endix)h(A.)150 1305 y Fg(Theorem)h(1)24 b Fp(A)13 b(r)n(andom)e(iter)n(ative)i(pr)n(o) n(c)n(ess)e Ft(\001)1047 1312 y Fm(n)p Fk(+1)1115 1305 y Ft(\()p Fo(x)p Ft(\))j(=)g(\(1)p Fv(\000)p Fo(\013)1360 1312 y Fm(n)1383 1305 y Ft(\()p Fo(x)p Ft(\)\)\001)1509 1312 y Fm(n)1532 1305 y Ft(\()p Fo(x)p Ft(\)+)p Fo(\014)1664 1312 y Fm(n)1686 1305 y Ft(\()p Fo(x)p Ft(\))p Fo(F)1784 1312 y Fm(n)1807 1305 y Ft(\()p Fo(x)p Ft(\))150 1411 y Fp(c)n(onver)n(ges)k(to)g(zer)n(o)f(w.p.1)g(under)h(the)g(fol)r (lowing)i(assumptions:)203 1541 y(1\))k Fo(x)13 b Fv(2)i Fo(S)s Fp(,)i(wher)n(e)g Fo(S)k Fp(is)c(a)g(\014nite)i(set.)203 1688 y(2\))272 1655 y Fj(P)316 1699 y Fm(n)348 1688 y Fo(\013)379 1695 y Fm(n)402 1688 y Ft(\()p Fo(x)p Ft(\))i(=)h Fv(1)p Fp(,)636 1655 y Fj(P)680 1699 y Fm(n)712 1688 y Fo(\013)743 1670 y Fk(2)743 1700 y Fm(n)767 1688 y Ft(\()p Fo(x)p Ft(\))f Fo(<)g Fv(1)p Fp(,)1001 1655 y Fj(P)1045 1699 y Fm(n)1076 1688 y Fo(\014)1104 1695 y Fm(n)1127 1688 y Ft(\()p Fo(x)p Ft(\))h(=)f Fv(1)p Fp(,)1362 1655 y Fj(P)1406 1699 y Fm(n)1437 1688 y Fo(\014)1468 1670 y Fk(2)1465 1700 y Fm(n)1488 1688 y Ft(\()p Fo(x)p Ft(\))h Fo(<)f Fv(1)p Fp(,)i(and)272 1797 y Ft(E)p Fv(f)p Fo(\014)358 1804 y Fm(n)381 1797 y Ft(\()p Fo(x)p Ft(\))p Fv(j)p Fo(P)492 1804 y Fm(n)515 1797 y Fv(g)14 b(\024)g Ft(E)p Fv(f)p Fo(\013)696 1804 y Fm(n)719 1797 y Ft(\()p Fo(x)p Ft(\))p Fv(j)p Fo(P)830 1804 y Fm(n)854 1797 y Fv(g)p Fp(uniformly)j(over)h Fo(x)f Fp(w.p.1.)203 1942 y(3\))24 b Fv(k)14 b Ft(E)p Fv(f)p Fo(F)401 1949 y Fm(n)424 1942 y Ft(\()p Fo(x)p Ft(\))p Fv(j)p Fo(P)535 1949 y Fm(n)558 1942 y Fo(;)8 b(\014)608 1949 y Fm(n)631 1942 y Fv(g)14 b(k)695 1949 y Fm(W)735 1942 y Fv(\024)g Fo(\015)i Fv(k)e Ft(\001)909 1949 y Fm(n)946 1942 y Fv(k)971 1949 y Fm(W)1011 1942 y Fp(,)k(wher)n(e)f Fo(\015)g Fv(2)d Ft(\(0)p Fo(;)8 b Ft(1\))p Fp(.)203 2089 y(4\))24 b Ft(V)l(ar)p Fv(f)p Fo(F)405 2096 y Fm(n)428 2089 y Ft(\()p Fo(x)p Ft(\))p Fv(j)p Fo(P)539 2096 y Fm(n)562 2089 y Fo(;)8 b(\014)612 2096 y Fm(n)635 2089 y Fv(g)14 b(\024)f Fo(C)t Ft(\(1+)h Fv(k)g Ft(\001)940 2096 y Fm(n)977 2089 y Fv(k)1002 2096 y Fm(W)1042 2089 y Ft(\))1061 2071 y Fk(2)1081 2089 y Fp(,)j(wher)n(e)h Fo(C)i Fp(is)e(some)f(c)n(onstant.)150 2222 y(Her)n(e)g Fo(P)297 2229 y Fm(n)335 2222 y Ft(=)d Fv(f)p Fo(X)452 2229 y Fm(n)475 2222 y Fo(;)8 b(X)537 2229 y Fm(n)p Fl(\000)p Fk(1)606 2222 y Fo(;)g(:)g(:)g(:)g(;)g(F)748 2229 y Fm(n)p Fl(\000)p Fk(1)816 2222 y Fo(;)g(:)g(:)g(:)f(;)h(\013)956 2229 y Fm(n)p Fl(\000)p Fk(1)1025 2222 y Fo(;)g(:)g(:)g(:)f(;)h(\014) 1162 2229 y Fm(n)p Fl(\000)p Fk(1)1230 2222 y Fo(;)g(:)g(:)g(:)o Fv(g)17 b Fp(stands)g(for)f(the)i(p)n(ast)e(at)150 2331 y(step)i Fo(n)p Fp(.)23 b Fo(F)350 2338 y Fm(n)373 2331 y Ft(\()p Fo(x)p Ft(\))p Fp(,)17 b Fo(\013)502 2338 y Fm(n)525 2331 y Ft(\()p Fo(x)p Ft(\))h Fp(and)f Fo(\014)731 2338 y Fm(n)754 2331 y Ft(\()p Fo(x)p Ft(\))h Fp(ar)n(e)f(al)r(lowe)n (d)h(to)g(dep)n(end)g(on)g(the)g(p)n(ast.)k Fo(\013)1616 2338 y Fm(n)1640 2331 y Ft(\()p Fo(x)p Ft(\))17 b Fp(and)150 2441 y Fo(\014)178 2448 y Fm(n)201 2441 y Ft(\()p Fo(x)p Ft(\))d Fp(ar)n(e)g(assume)n(d)g(to)g(b)n(e)h(non-ne)n(gative)i(and)e (mutual)r(ly)g(indep)n(endent)i(given)f Fo(P)1658 2448 y Fm(n)1681 2441 y Fp(.)22 b(The)150 2550 y(notation)c Fv(k)c(\001)g(k)436 2557 y Fm(W)493 2550 y Fp(r)n(efers)j(to)h(some)f (weighte)n(d)i(maximum)e(norm.)951 2775 y Ft(10)p eop %%Page: 11 11 11 10 bop 248 192 a Ft(In)16 b(applying)h(the)f(theorem,)f(the)h(\001) 922 199 y Fm(n)962 192 y Ft(pro)q(cess)i(will)d(generally)h(represen)o (t)g(the)g(dif-)150 298 y(ference)f(b)q(et)o(w)o(een)h(a)h(sto)q(c)o (hastic)g(pro)q(cess)g(of)g(in)o(terest)e(and)i(some)f(optimal)f(v)m (alue)h(\(e.g.,)150 407 y(the)d(optimal)e(v)m(alue)i(function\).)20 b(The)12 b(form)o(ulation)g(of)h(the)g(theorem)e(therefore)h(requires) 150 514 y(kno)o(wledge)j(to)i(b)q(e)f(a)o(v)m(ailable)f(ab)q(out)i(the) f(optimal)f(solution)h(to)g(the)g(learning)g(problem)150 618 y(b)q(efore)22 b(it)f(can)h(b)q(e)g(applied)f(to)h(an)o(y)g (algorithm)e(whose)j(con)o(v)o(ergence)d(is)h(to)h(b)q(e)g(v)o(eri-)150 721 y(\014ed.)f(In)14 b(the)g(case)h(of)g(Q-learning)f(the)h(required)e (kno)o(wledge)h(is)h(a)o(v)m(ailable)f(through)h(the)150 825 y(theory)h(of)h(DP)f(and)h(Bellman's)d(equation)i(in)g(particular.) 248 929 y(The)11 b(con)o(v)o(ergence)f(of)i(the)f(Q-learning)h (algorithm)e(no)o(w)i(follo)o(ws)g(easily)e(b)o(y)h(relating)150 1038 y(the)16 b(algorithm)f(to)h(the)g(con)o(v)o(erging)f(sto)q(c)o (hastic)h(pro)q(cess)g(de\014ned)g(b)o(y)f(Theorem)g(1.)1714 1020 y Fk(1)1755 1038 y Ft(In)150 1141 y(the)h(form)f(of)i(the)f (theorem)e(w)o(e)i(ha)o(v)o(e:)150 1290 y Fg(Theorem)h(2)24 b Fp(The)18 b(Q-le)n(arning)h(algorithm)e(given)i(by)242 1446 y Fo(Q)281 1453 y Fm(t)p Fk(+1)340 1446 y Ft(\()p Fo(s)382 1453 y Fm(t)397 1446 y Fo(;)8 b(u)447 1453 y Fm(t)462 1446 y Ft(\))13 b(=)h(\(1)e Fv(\000)e Fo(\013)681 1453 y Fm(t)696 1446 y Ft(\()p Fo(s)738 1453 y Fm(t)753 1446 y Fo(;)e(u)803 1453 y Fm(t)817 1446 y Ft(\)\))p Fo(Q)894 1453 y Fm(t)909 1446 y Ft(\()p Fo(s)951 1453 y Fm(t)966 1446 y Fo(;)g(u)1016 1453 y Fm(t)1030 1446 y Ft(\))j(+)g Fo(\013)1140 1453 y Fm(t)1155 1446 y Ft(\()p Fo(s)1197 1453 y Fm(t)1212 1446 y Fo(;)d(u)1262 1453 y Fm(t)1276 1446 y Ft(\)[)p Fo(c)1330 1453 y Fm(s)1346 1457 y Fi(t)1362 1446 y Ft(\()p Fo(u)1409 1453 y Fm(t)1423 1446 y Ft(\))k(+)f Fo(\015)s(V)1559 1453 y Fm(t)1574 1446 y Ft(\()p Fo(s)1616 1453 y Fm(t)p Fk(+1)1676 1446 y Ft(\)])150 1605 y Fp(c)n(onver)n(ges)18 b(to)g(the)g(optimal)f Fo(Q)721 1587 y Fl(\003)741 1605 y Ft(\()p Fo(s;)8 b(u)p Ft(\))17 b Fp(values)h(if)203 1756 y(1\))24 b(The)18 b(state)g(and)f(action)h(sp)n(ac)n(es)f(ar)n(e)g(\014nite.)203 1909 y(2\))272 1876 y Fj(P)316 1919 y Fm(t)339 1909 y Fo(\013)370 1916 y Fm(t)385 1909 y Ft(\()p Fo(s;)8 b(u)p Ft(\))13 b(=)h Fv(1)j Fp(and)723 1876 y Fj(P)767 1919 y Fm(t)790 1909 y Fo(\013)821 1891 y Fk(2)821 1921 y Fm(t)841 1909 y Ft(\()p Fo(s;)8 b(u)p Ft(\))13 b Fo(<)h Fv(1)j Fp(uniformly)g(over)h Fo(s)g Fp(and)f Fo(u)g Fp(w.p.1.)203 2060 y(3\))24 b Ft(V)l(ar)p Fv(f)p Fo(c)394 2067 y Fm(s)412 2060 y Ft(\()p Fo(u)p Ft(\))p Fv(g)17 b Fp(is)g(\014nite.)203 2210 y(3\))24 b(If)17 b Fo(\015)g Ft(=)d(1)k Fp(al)r(l)g(p)n(olicies)g (le)n(ad)g(to)f(a)h(c)n(ost)f(fr)n(e)n(e)g(terminal)h(state)g(w.p.1.)p 150 2257 660 2 v 206 2287 a Ff(1)225 2303 y Fe(W)m(e)13 b(note)h(that)g(the)h(theorem)f(is)f(more)g(p)q(o)o(w)o(erful)h(than)g (is)f(needed)j(to)d(pro)o(v)o(e)h(the)h(con)o(v)o(ergence)h(of)150 2401 y(Q-learning.)j(Its)c(generalit)o(y)m(,)f(ho)o(w)o(ev)o(er,)g (allo)o(ws)f(it)h(to)h(b)q(e)g(applied)f(to)g(other)h(algorithms)d(as)j (w)o(ell)f(\(see)150 2513 y(the)g(follo)o(wing)e(section)i(on)g(TD\()p Fd(\025)p Fe(\).)951 2775 y Ft(11)p eop %%Page: 12 12 12 11 bop 248 347 a Fg(Pro)r(of.)23 b Ft(By)16 b(subtracting)i Fo(Q)798 329 y Fl(\003)817 347 y Ft(\()p Fo(s;)8 b(u)p Ft(\))17 b(from)f(b)q(oth)h(sides)g(of)h(the)e(learning)h(rule)f(and) 150 457 y(b)o(y)g(de\014ning)g(\001)443 464 y Fm(t)457 457 y Ft(\()p Fo(s;)8 b(u)p Ft(\))14 b(=)g Fo(Q)673 464 y Fm(t)687 457 y Ft(\()p Fo(s;)8 b(u)p Ft(\))j Fv(\000)f Fo(Q)897 439 y Fl(\003)917 457 y Ft(\()p Fo(s;)e(u)p Ft(\))16 b(together)g(with)564 607 y Fo(F)596 614 y Fm(t)610 607 y Ft(\()p Fo(s;)8 b(u)p Ft(\))14 b(=)f Fo(c)807 614 y Fm(s)826 607 y Ft(\()p Fo(u)p Ft(\))e(+)g Fo(\015)s(V)1008 614 y Fm(t)1023 607 y Ft(\()p Fo(s)1065 614 y Fm(next)1137 607 y Ft(\))g Fv(\000)g Fo(Q)1256 586 y Fl(\003)1276 607 y Ft(\()p Fo(s;)d(u)p Ft(\))326 b(\(12\))150 754 y(the)16 b(Q-learning)h(algorithm)e(can)i(b)q(e)f(seen)g(to)h(ha)o(v)o (e)f(the)g(form)g(of)g(the)h(pro)q(cess)g(in)f(The-)150 860 y(orem)f(1)i(with)f Fo(\014)452 867 y Fm(t)466 860 y Ft(\()p Fo(s;)8 b(u)p Ft(\))13 b(=)h Fo(\013)673 867 y Fm(t)688 860 y Ft(\()p Fo(s;)8 b(u)p Ft(\).)248 970 y(T)l(o)19 b(v)o(erify)f(that)i Fo(F)602 977 y Fm(t)616 970 y Ft(\()p Fo(s;)8 b(u)p Ft(\))19 b(has)h(the)f(required)g(prop)q (erties)g(w)o(e)g(b)q(egin)g(b)o(y)g(sho)o(wing)150 1076 y(that)f(it)f(is)h(a)g(con)o(traction)f(mapping)g(with)h(resp)q(ect)f (to)h(some)f(maxim)o(um)c(norm.)25 b(This)150 1180 y(is)15 b(done)g(b)o(y)g(relating)f Fo(F)589 1187 y Fm(t)619 1180 y Ft(to)h(the)g(DP)g(v)m(alue)g(iteration)g(op)q(erator)h(for)f (the)g(same)f(Mark)o(o)o(v)150 1284 y(c)o(hain.)21 b(More)16 b(sp)q(eci\014cally)l(,)313 1439 y(max)348 1464 y Fm(u)412 1439 y Fv(j)p Ft(E)p Fv(f)p Fo(F)516 1446 y Fm(t)530 1439 y Ft(\()p Fo(i;)8 b(u)p Ft(\))p Fv(gj)41 b Ft(=)h Fo(\015)11 b Ft(max)866 1464 y Fm(u)930 1439 y Fv(j)952 1398 y Fj(X)974 1489 y Fm(j)1020 1439 y Fo(p)1044 1446 y Fm(ij)1075 1439 y Ft(\()p Fo(u)p Ft(\)[)p Fo(V)1183 1446 y Fm(t)1198 1439 y Ft(\()p Fo(j)s Ft(\))g Fv(\000)g Fo(V)1359 1419 y Fl(\003)1379 1439 y Ft(\()p Fo(j)s Ft(\)])p Fv(j)714 1617 y(\024)42 b Fo(\015)11 b Ft(max)866 1642 y Fm(u)930 1576 y Fj(X)952 1667 y Fm(j)998 1617 y Fo(p)1022 1624 y Fm(ij)1053 1617 y Ft(\()p Fo(u)p Ft(\))d(max)1163 1642 y Fm(v)1226 1617 y Fv(j)p Fo(Q)1279 1624 y Fm(t)1293 1617 y Ft(\()p Fo(j;)g(v)r Ft(\))j Fv(\000)g Fo(Q)1499 1597 y Fl(\003)1518 1617 y Ft(\()p Fo(j;)d(v)r Ft(\))p Fv(j)715 1795 y Ft(=)42 b Fo(\015)11 b Ft(max)866 1820 y Fm(u)930 1754 y Fj(X)952 1845 y Fm(j)998 1795 y Fo(p)1022 1802 y Fm(ij)1053 1795 y Ft(\()p Fo(u)p Ft(\))p Fo(V)1158 1775 y Fk(\001)1190 1795 y Ft(\()p Fo(j)s Ft(\))i(=)h Fo(T)7 b Ft(\()p Fo(V)1410 1775 y Fk(\001)1441 1795 y Ft(\)\()p Fo(i)p Ft(\))150 1996 y(where)17 b(w)o(e)f(ha)o(v)o(e)h(used) g(the)f(notation)i Fo(V)910 1978 y Fk(\001)941 1996 y Ft(\()p Fo(j)s Ft(\))d(=)g(max)1161 2003 y Fm(v)1190 1996 y Fv(j)p Fo(Q)1243 2003 y Fm(t)1257 1996 y Ft(\()p Fo(j;)8 b(v)r Ft(\))j Fv(\000)h Fo(Q)1464 1978 y Fl(\003)1483 1996 y Ft(\()p Fo(j;)c(v)r Ft(\))p Fv(j)16 b Ft(and)i Fo(T)23 b Ft(is)150 2103 y(the)18 b(DP)h(v)m(alue)f(iteration)g(op)q (erator)i(for)e(the)g(case)h(where)f(the)g(costs)h(asso)q(ciated)g (with)150 2209 y(eac)o(h)g(state)g(are)g(zero)g(\(cf.)30 b(Equation)19 b(\(4\)\).)31 b(If)18 b Fo(\015)k(<)d Ft(1)h(the)f(con)o (traction)g(prop)q(ert)o(y)g(of)150 2322 y(E)p Fv(f)p Fo(F)240 2329 y Fm(t)254 2322 y Ft(\()p Fo(i;)8 b(u)p Ft(\))p Fv(g)17 b Ft(can)g(b)q(e)g(seen)f(from)g(the)h(fourth)g(form)o (ula)f(b)o(y)g(b)q(ounding)1478 2289 y Fj(P)1521 2332 y Fm(j)1548 2322 y Fo(p)1572 2329 y Fm(ij)1603 2322 y Ft(\()p Fo(u)p Ft(\))p Fo(V)1708 2304 y Fk(\001)1739 2322 y Ft(\()p Fo(j)s Ft(\))150 2439 y(b)o(y)k(max)313 2446 y Fm(j)339 2439 y Fo(V)378 2421 y Fk(\001)410 2439 y Ft(\()p Fo(j)s Ft(\))g(and)h(then)f(including)g(the)g Fo(\015)j Ft(factor.)34 b(When)20 b(the)g(future)g(costs)h(are)150 2550 y(not)c(discoun)o(ted)g(\()p Fo(\015)g Ft(=)e(1\))i(but)g(the)g(c) o(hain)f(is)h(absorbing)h(and)f(all)g(p)q(olicies)f(lead)g(to)h(the)951 2775 y(12)p eop %%Page: 13 13 13 12 bop 150 192 a Ft(terminal)13 b(state)j(w.p.1)f(there)g(still)f (exists)h(a)h(w)o(eigh)o(ted)e(maxim)o(um)d(norm)j(with)h(resp)q(ect) 150 298 y(to)20 b(whic)o(h)f Fo(T)26 b Ft(is)20 b(a)g(con)o(traction)g (mapping)f(\(see)g(e.g.)31 b(Bertsek)m(as)20 b(&)f(Tsitsiklis,)g (1989\))150 407 y(thereb)o(y)e(forcing)g(the)h(con)o(traction)f(of)h(E) p Fv(f)p Fo(F)980 414 y Fm(t)995 407 y Ft(\()p Fo(i;)8 b(u)p Ft(\))p Fv(g)p Ft(.)24 b(The)18 b(v)m(ariance)f(of)h Fo(F)1546 414 y Fm(t)1561 407 y Ft(\()p Fo(s;)8 b(u)p Ft(\))17 b(giv)o(en)150 517 y(the)f(past)g(is)g(within)g(the)f(b)q (ounds)j(of)e(Theorem)e(1)j(as)f(it)g(dep)q(ends)g(on)h Fo(Q)1497 524 y Fm(t)1511 517 y Ft(\()p Fo(s;)8 b(u)p Ft(\))15 b(at)i(most)150 626 y(linearly)e(and)i(the)f(v)m(ariance)g(of) g Fo(c)771 633 y Fm(s)790 626 y Ft(\()p Fo(u)p Ft(\))g(is)g(b)q (ounded.)248 733 y(Note)f(that)i(the)f(pro)q(of)h(co)o(v)o(ers)f(b)q (oth)h(the)f(on-line)g(and)h(batc)o(h)f(v)o(ersions.)164 b Fc(2)150 986 y Fw(The)28 b(TD\()p Fb(\025)p Fw(\))g(algorithm)150 1150 y Ft(The)15 b(TD\()p Fo(\025)p Ft(\))g(\(Sutton,)g(1988\))i(is)d (also)h(a)h(DP-based)f(learning)g(algorithm)f(that)h(is)f(natu-)150 1257 y(rally)i(de\014ned)g(in)g(a)h(Mark)o(o)o(v)f(en)o(vironmen)o(t.)k (Unlik)o(e)14 b(Q-learning,)i(ho)o(w)o(ev)o(er,)f(TD)i(do)q(es)150 1361 y(not)c(in)o(v)o(olv)o(e)d(decision-making)h(tasks)i(but)f(rather) h(predictions)e(ab)q(out)j(the)e(future)g(costs)150 1467 y(of)18 b(an)g(ev)o(olving)e(system.)23 b(TD\()p Fo(\025)p Ft(\))18 b(con)o(v)o(erges)f(to)h(the)f(same)f(predictions)h(as)h(a)g (v)o(ersion)150 1574 y(of)j(Q-learning)g(in)g(whic)o(h)g(there)f(is)h (only)g(one)g(action)g(a)o(v)m(ailable)g(at)g(eac)o(h)g(state,)h(but) 150 1678 y(the)14 b(algorithms)f(are)h(deriv)o(ed)e(from)h(sligh)o(tly) g(di\013eren)o(t)g(grounds)i(and)g(their)e(b)q(eha)o(vioral)150 1781 y(di\013erences)g(are)h(not)g(w)o(ell)e(understo)q(o)q(d.)22 b(In)14 b(this)g(section)f(w)o(e)g(in)o(tro)q(duce)h(the)f(algorithm) 150 1885 y(and)k(its)f(deriv)m(ation.)21 b(The)16 b(pro)q(of)h(of)g (con)o(v)o(ergence)e(is)h(giv)o(en)f(in)h(the)g(follo)o(wing)g (section.)248 1992 y(Let)c(us)h(de\014ne)g Fo(V)556 1999 y Fm(t)571 1992 y Ft(\()p Fo(i)p Ft(\))f(to)h(b)q(e)g(the)f(curren)o(t) g(estimate)f(of)i(the)f(exp)q(ected)g(cost)h(incurred)150 2098 y(during)24 b(the)f(ev)o(olution)f(of)i(the)f(system)f(starting)h (from)g(state)g Fo(i)g Ft(and)h(let)f Fo(c)1622 2105 y Fm(i)1659 2098 y Ft(denote)150 2202 y(the)e(instan)o(taneous)i (random)e(cost)h(at)f(state)h Fo(i)p Ft(.)37 b(As)21 b(in)g(the)h(case)f(of)h(Q-learning)f(w)o(e)150 2306 y(assume)16 b(that)h(the)f(future)g(costs)h(are)f(discoun)o(ted)g(at)h (eac)o(h)f(state)g(b)o(y)g(a)h(factor)g Fo(\015)s Ft(.)22 b(If)16 b(no)150 2412 y(discoun)o(ting)i(tak)o(es)f(place)h(\()p Fo(\015)i Ft(=)c(1\))j(w)o(e)e(need)h(to)g(assume)f(that)i(the)f(Mark)o (o)o(v)f(c)o(hain)g(is)150 2519 y(absorbing,)h(that)f(is,)f(there)h (exists)f(a)h(cost)g(free)g(terminal)d(state)j(to)h(whic)o(h)e(the)g (system)951 2775 y(13)p eop %%Page: 14 14 14 13 bop 150 192 a Ft(con)o(v)o(erges)15 b(with)i(probabilit)o(y)e (one.)248 295 y(W)l(e)f(are)g(concerned)g(with)g(estimating)f(the)h (future)h(costs)g(that)f(the)h(learner)e(has)j(to)150 399 y(incur.)k(One)14 b(w)o(a)o(y)g(to)g(ac)o(hiev)o(e)e(these)i (predictions)f(is)h(to)h(simply)d(observ)o(e)h Fo(n)i Ft(consecutiv)o(e)150 503 y(random)g(costs)h(w)o(eigh)o(ted)f(b)o(y)g (the)g(discoun)o(t)g(factor)h(and)g(to)g(add)g(the)f(b)q(est)h (estimate)e(of)150 607 y(the)i(costs)h(thereafter.)j(This)d(giv)o(es)e (us)i(the)f(estimate)270 768 y Fo(V)309 743 y Fk(\()p Fm(n)p Fk(\))298 778 y Fm(t)360 768 y Ft(\()p Fo(i)396 775 y Fm(t)411 768 y Ft(\))e(=)f Fo(c)516 775 y Fm(i)528 779 y Fi(t)555 768 y Ft(+)e Fo(\015)s(c)653 775 y Fm(i)665 780 y Fi(t)p Fh(+1)731 768 y Ft(+)g Fo(\015)808 748 y Fk(2)828 768 y Fo(c)849 775 y Fm(i)861 780 y Fi(t)p Fh(+2)926 768 y Ft(+)g Fo(:)d(:)g(:)j Ft(+)g Fo(\015)1121 748 y Fm(n)p Fl(\000)p Fk(1)1189 768 y Fo(c)1210 775 y Fm(i)1222 780 y Fi(t)p Fh(+)p Fi(n)p Fa(\000)p Fh(1)1331 768 y Ft(+)g Fo(\015)1408 748 y Fm(n)1432 768 y Fo(V)1460 775 y Fm(t)1475 768 y Ft(\()p Fo(i)1511 775 y Fm(t)p Fk(+)p Fm(n)1574 768 y Ft(\))120 b(\(13\))150 920 y(The)16 b(exp)q(ected)f(v)m (alue)g(of)i(this)e(can)h(b)q(e)g(sho)o(wn)h(to)f(b)q(e)g(a)g(strictly) f(b)q(etter)g(estimate)f(than)150 1026 y(the)k(curren)o(t)f(estimate)f (is)i(\(W)l(atkins,)g(1989\).)29 b(In)18 b(the)g(undiscoun)o(ted)g (case)g(this)g(holds)150 1133 y(only)h(when)g Fo(n)g Ft(is)f(larger)h(than)h(some)e(c)o(hain-dep)q(enden)o(t)g(constan)o(t.) 29 b(T)l(o)20 b(demonstrate)150 1252 y(this)12 b(let)g(us)h(replace)e Fo(V)556 1259 y Fm(t)584 1252 y Ft(with)h Fo(V)730 1234 y Fl(\003)762 1252 y Ft(in)g(the)g(ab)q(o)o(v)o(e)h(form)o(ula)e (giving)h(E)p Fv(f)p Fo(V)1443 1227 y Fl(\003)p Fk(\()p Fm(n)p Fk(\))1432 1262 y Fm(t)1512 1252 y Ft(\()p Fo(i)1548 1259 y Fm(t)1562 1252 y Ft(\))p Fv(g)i Ft(=)g Fo(V)1711 1234 y Fl(\003)1731 1252 y Ft(\()p Fo(i)1767 1259 y Fm(t)1781 1252 y Ft(\))150 1359 y(whic)o(h)i(implies)228 1520 y(max)267 1549 y Fm(i)327 1520 y Fv(j)p Ft(E)p Fv(f)p Fo(V)438 1495 y Fk(\()p Fm(n)p Fk(\))427 1530 y Fm(t)489 1520 y Ft(\()p Fo(i)p Ft(\))p Fv(g)11 b(\000)f Fo(V)669 1500 y Fl(\003)688 1520 y Ft(\()p Fo(i)p Ft(\))p Fv(j)k(\024)f Fo(\015)851 1500 y Fm(n)883 1520 y Ft(max)922 1549 y Fm(i)982 1520 y Ft(Pr)p Fv(f)p Fo(m)1102 1527 y Fm(i)1130 1520 y Fv(\025)g Fo(n)p Fv(g)8 b Ft(max)1284 1549 y Fm(i)1344 1520 y Fv(j)p Fo(V)1386 1527 y Fm(t)1401 1520 y Ft(\()p Fo(i)p Ft(\))j Fv(\000)f Fo(V)1556 1500 y Fl(\003)1575 1520 y Ft(\()p Fo(i)p Ft(\))p Fv(j)69 b Ft(\(14\))150 1692 y(where)13 b Fo(m)331 1699 y Fm(i)359 1692 y Ft(is)g(the)g(n)o(um) o(b)q(er)f(of)i(steps)g(in)f(a)h(sequence)e(that)i(b)q(egins)g(in)g (state)g Fo(i)f Ft(\(in\014nite)f(in)150 1801 y(the)j(non-absorbing)i (case\).)j(This)15 b(implies)d(that)k(if)e(either)g Fo(\015)j(<)c Ft(1)j(or)f Fo(n)g Ft(is)f(large)h(enough)150 1908 y(so)e(that)g(the)f (c)o(hain)g(can)g(terminate)e(b)q(efore)j Fo(n)f Ft(steps)h(starting)g (from)e(an)i(arbitrary)f(initial)150 2028 y(state)19 b(then)f(the)g(estimate)e Fo(V)708 2002 y Fk(\()p Fm(n)p Fk(\))696 2037 y Fm(t)777 2028 y Ft(is)i(strictly)f(b)q(etter)h(than)h Fo(V)1288 2035 y Fm(t)1303 2028 y Ft(.)28 b(In)18 b(general,)g(the)g (larger)150 2132 y Fo(n)23 b Ft(the)g(more)f(un)o(biased)h(the)f (estimate)g(is)g(as)i(the)f(e\013ect)f(of)i(incorrect)e Fo(V)1573 2139 y Fm(t)1611 2132 y Ft(v)m(anishes.)150 2233 y(Ho)o(w)o(ev)o(er,)f(larger)h Fo(n)g Ft(increases)g(the)g(v)m (ariance)g(of)g(the)g(estimate)e(as)j(there)e(are)h(more)150 2340 y(\(indep)q(enden)o(t\))15 b(terms)g(in)h(the)g(sum.)248 2446 y(Despite)i(the)h(error)g(reduction)f(prop)q(ert)o(y)h(of)h(the)f (truncated)g(estimate)e(it)h(is)h(dif-)150 2550 y(\014cult)i(to)g (calculate)f(in)h(practice)f(as)i(one)f(w)o(ould)h(ha)o(v)o(e)e(to)h(w) o(ait)g Fo(n)h Ft(steps)f(b)q(efore)g(the)951 2775 y(14)p eop %%Page: 15 15 15 14 bop 150 192 a Ft(predictions)18 b(could)h(b)q(e)h(up)q(dated.)31 b(In)18 b(addition)i(it)e(clearly)g(has)i(a)g(h)o(uge)f(v)m(ariance.)29 b(A)150 295 y(remedy)12 b(to)j(these)f(problems)g(is)g(obtained)h(b)o (y)f(constructing)h(a)f(new)h(estimate)e(b)o(y)h(a)o(v)o(er-)150 402 y(aging)i(o)o(v)o(er)f(the)g(truncated)h(predictions.)k(TD\()p Fo(\025)p Ft(\))d(is)e(based)h(on)g(taking)g(the)f(geometric)150 495 y(a)o(v)o(erage:)646 637 y Fo(V)685 616 y Fm(\025)674 649 y(t)708 637 y Ft(\()p Fo(i)p Ft(\))e(=)h(\(1)d Fv(\000)g Fo(\025)p Ft(\))1003 583 y Fl(1)991 595 y Fj(X)988 686 y Fm(n)p Fk(=1)1063 637 y Fo(\025)1091 616 y Fm(n)p Fl(\000)p Fk(1)1160 637 y Fo(V)1199 611 y Fk(\()p Fm(n)p Fk(\))1188 647 y Fm(t)1250 637 y Ft(\()p Fo(i)p Ft(\))408 b(\(15\))150 809 y(As)23 b(a)h(w)o(eigh)o(ted)f(a)o(v)o(erage)g(it)g(is)h(still)e(a) i(strictly)e(b)q(etter)i(estimate)e(than)i Fo(V)1612 816 y Fm(t)1627 809 y Ft(\()p Fo(i)p Ft(\))f(with)150 919 y(the)g(additional)g(b)q(ene\014t)g(of)g(b)q(eing)g(b)q(etter)f(in) h(the)g(undiscoun)o(ted)f(case)h(as)h(w)o(ell)d(\(as)150 1028 y(the)d(summation)e(extends)h(to)i(in\014nit)o(y\).)25 b(F)l(urthermore,)16 b(w)o(e)i(ha)o(v)o(e)f(in)o(tro)q(duced)h(a)g(new) 150 1134 y(parameter)f Fo(\025)h Ft(whic)o(h)f(a\013ects)h(the)g (trade-o\013)h(b)q(et)o(w)o(een)e(the)h(bias)g(and)g(v)m(ariance)g(of)g (the)150 1241 y(estimate)11 b(\(W)l(atkins,)j(1989\).)22 b(An)12 b(increase)h(in)g Fo(\025)h Ft(puts)f(more)f(w)o(eigh)o(t)h(on) h(less)f(biased)g(es-)150 1354 y(timates)f(with)h(higher)g(v)m (ariances)h(and)g(th)o(us)f(the)g(bias)h(in)f Fo(V)1251 1336 y Fm(\025)1240 1366 y(t)1287 1354 y Ft(decreases)h(at)f(the)g(exp) q(ense)150 1460 y(of)k(a)f(higher)g(v)m(ariance.)248 1564 y(The)d(mathematical)d(con)o(v)o(enience)h(of)i(using)h(the)f (geometric)e(a)o(v)o(erage)i(can)g(b)q(e)g(seen)150 1675 y(as)g(follo)o(ws.)19 b(Giv)o(en)11 b(the)h(estimates)e Fo(V)849 1656 y Fm(\025)838 1687 y(t)872 1675 y Ft(\()p Fo(i)p Ft(\))i(the)f(ob)o(vious)h(w)o(a)o(y)g(to)g(use)g(them)e(in)i(a) g(learning)150 1781 y(rule)k(is)595 1884 y Fo(V)623 1891 y Fm(t)p Fk(+1)684 1884 y Ft(\()p Fo(i)720 1891 y Fm(t)734 1884 y Ft(\))e(=)g Fo(V)847 1891 y Fm(t)862 1884 y Ft(\()p Fo(i)898 1891 y Fm(t)912 1884 y Ft(\))d(+)g Fo(\013)p Ft([)p Fo(V)1076 1864 y Fm(\025)1064 1897 y(t)1098 1884 y Ft(\()p Fo(i)1134 1891 y Fm(t)1149 1884 y Ft(\))g Fv(\000)f Fo(V)1256 1891 y Fm(t)1272 1884 y Ft(\()p Fo(i)1308 1891 y Fm(t)1322 1884 y Ft(\)])358 b(\(16\))150 2013 y(In)16 b(terms)f(of)h Fp(pr)n(e)n(diction)h(di\013er)n(enc)n(es)p Ft(,)f(that)h(is)646 2159 y(\001)687 2166 y Fm(t)701 2159 y Ft(\()p Fo(i)737 2166 y Fm(t)751 2159 y Ft(\))d(=)g Fo(c)857 2166 y Fm(i)869 2170 y Fi(t)896 2159 y Ft(+)d Fo(\015)s(V)1001 2166 y Fm(t)1016 2159 y Ft(\()p Fo(i)1052 2166 y Fm(t)p Fk(+1)1112 2159 y Ft(\))g Fv(\000)g Fo(V)1220 2166 y Fm(t)1235 2159 y Ft(\()p Fo(i)1271 2166 y Fm(t)1285 2159 y Ft(\))409 b(\(17\))150 2304 y(the)15 b(geometric)f(w)o(eigh)o (ting)h(allo)o(ws)g(us)h(to)g(write)f(the)h(correction)f(term)f(in)h (the)g(learning)150 2408 y(rule)h(as)330 2550 y Fo(V)370 2529 y Fm(\025)358 2562 y(t)392 2550 y Ft(\()p Fo(i)428 2557 y Fm(t)443 2550 y Ft(\))11 b Fv(\000)f Fo(V)550 2557 y Fm(t)566 2550 y Ft(\()p Fo(i)602 2557 y Fm(t)616 2550 y Ft(\))k(=)g(\001)742 2557 y Fm(t)756 2550 y Ft(\()p Fo(i)792 2557 y Fm(t)807 2550 y Ft(\))d(+)g(\()p Fo(\025\015)s Ft(\)\001)1021 2557 y Fm(t)1036 2550 y Ft(\()p Fo(i)1072 2557 y Fm(t)p Fk(+1)1131 2550 y Ft(\))g(+)g(\()p Fo(\025\015)s Ft(\))1304 2529 y Fk(2)1324 2550 y Ft(\001)1365 2557 y Fm(t)1380 2550 y Ft(\()p Fo(i)1416 2557 y Fm(t)p Fk(+2)1475 2550 y Ft(\))g(+)g Fo(:)d(:)g(:)101 b Ft(\(18\))951 2775 y(15)p eop %%Page: 16 16 16 15 bop 150 192 a Ft(Note)12 b(that)g(up)h(to)f(no)o(w)h(the)f (prediction)f(di\013erences)g(that)i(need)e(to)i(b)q(e)f(calculated)f (in)h(the)150 298 y(future)17 b(dep)q(end)h(on)g(the)f(curren)o(t)g Fo(V)820 305 y Fm(t)835 298 y Ft(\()p Fo(i)p Ft(\).)25 b(If)17 b(the)g(c)o(hain)g(is)g(nonabsorbing)i(this)f(compu-)150 405 y(tational)d(implausibilit)o(y)c(can,)k(ho)o(w)o(ev)o(er,)e(b)q(e)i (o)o(v)o(ercome)d(b)o(y)i(up)q(dating)i(the)f(predictions)150 508 y(at)21 b(eac)o(h)g(step)g(with)g(the)f(prediction)g(di\013erences) g(calculated)g(b)o(y)h(using)g(the)g(curren)o(t)150 615 y(predictions.)g(This)16 b(pro)q(cedure)g(giv)o(es)g(the)g(on-line)g(v) o(ersion)f(of)i(TD\()p Fo(\025)p Ft(\):)537 804 y Fo(V)565 811 y Fm(t)p Fk(+1)626 804 y Ft(\()p Fo(i)p Ft(\))c(=)h Fo(V)774 811 y Fm(t)789 804 y Ft(\()p Fo(i)p Ft(\))d(+)g Fo(\013)935 811 y Fm(t)950 804 y Ft(\001)991 811 y Fm(t)1005 804 y Ft(\()p Fo(i)1041 811 y Fm(t)1056 804 y Ft(\))1109 750 y Fm(t)1085 762 y Fj(X)1083 855 y Fm(k)q Fk(=0)1147 804 y Ft(\()p Fo(\015)s(\025)p Ft(\))1241 783 y Fm(t)p Fl(\000)p Fm(k)1303 804 y Fo(\037)1334 811 y Fm(i)1348 804 y Ft(\()p Fo(k)r Ft(\))300 b(\(19\))150 998 y(where)15 b Fo(\037)321 1005 y Fm(i)335 998 y Ft(\()p Fo(k)r Ft(\))h(is)f(the)g (indicator)h(v)m(ariable)f(of)h(whether)f(state)h Fo(i)g Ft(w)o(as)g(visited)e(at)i Fo(k)1662 980 y Fm(th)1713 998 y Ft(step)150 1107 y(\(of)g(a)h(sequence\).)j(Note)15 b(that)i(the)f(sum)f(con)o(tains)h(the)g(e\013ect)f(of)h(the)g(mo)q (di\014cations)g(or)150 1214 y(activit)o(y)f(traces)i(initiated)f(at)h (past)h(time)d(steps.)24 b(Moreo)o(v)o(er,)15 b(it)i(is)f(imp)q(ortan)o (t)h(to)g(note)150 1317 y(that)e(in)g(this)f(case)h(the)f (theoretically)f(desirable)h(prop)q(erties)h(of)g(the)f(estimates)g (deriv)o(ed)150 1424 y(earlier)19 b(ma)o(y)f(hold)i(only)f (asymptotically)f(\(see)h(the)h(con)o(v)o(ergence)e(pro)q(of)j(in)e (the)h(next)150 1533 y(section\).)248 1643 y(In)h(the)h(absorbing)i (case)e(the)g(estimates)f Fo(V)1081 1650 y Fm(t)1096 1643 y Ft(\()p Fo(i)p Ft(\))h(can)g(also)h(b)q(e)f(up)q(dated)h (o\013-line,)150 1749 y(that)d(is,)h(after)e(a)i(complete)d(sequence)g (has)j(b)q(een)f(observ)o(ed.)32 b(The)20 b(learning)f(rule)h(for)150 1853 y(this)h(case)g(is)g(deriv)o(ed)e(simply)g(from)h(collecting)g (the)g(correction)h(traces)g(initiated)f(at)150 1957 y(eac)o(h)e(step)g(of)h(the)f(sequence.)26 b(More)19 b(concisely)l(,)e(the)h(total)g(correction)g(is)g(the)g(sum)g(of)150 2063 y(individual)g(correction)h(traces)g(illustrated)g(in)g(Equation)h (\(18\).)31 b(This)20 b(results)f(in)g(the)150 2170 y(batc)o(h)d (learning)g(rule)482 2356 y Fo(V)510 2363 y Fm(n)p Fk(+1)579 2356 y Ft(\()p Fo(i)p Ft(\))d(=)h Fo(V)727 2363 y Fm(n)751 2356 y Ft(\()p Fo(i)p Ft(\))c(+)h Fo(\013)896 2363 y Fm(n)943 2302 y(m)928 2315 y Fj(X)929 2405 y Fm(t)p Fk(=1)997 2356 y Ft(\001)1038 2363 y Fm(n)1061 2356 y Ft(\()p Fo(i)1097 2363 y Fm(t)1111 2356 y Ft(\))1164 2302 y Fm(t)1141 2315 y Fj(X)1138 2407 y Fm(k)q Fk(=0)1203 2356 y Ft(\()p Fo(\015)s(\025)p Ft(\))1297 2335 y Fm(t)p Fl(\000)p Fm(k)1359 2356 y Fo(\037)1390 2363 y Fm(i)1403 2356 y Ft(\()p Fo(k)r Ft(\))245 b(\(20\))150 2550 y(where)16 b(the)g(\()p Fo(m)11 b Ft(+)g(1\))540 2532 y Fm(th)591 2550 y Ft(step)17 b(is)f(the)g(termination)e(state.) 951 2775 y(16)p eop %%Page: 17 17 17 16 bop 248 192 a Ft(W)l(e)19 b(note)h(that)g(the)g(ab)q(o)o(v)o(e)f (deriv)m(ation)h(of)g(the)f(TD\()p Fo(\025)p Ft(\))i(algorithm)e (corresp)q(onds)150 301 y(to)h(the)g(sp)q(eci\014c)g(c)o(hoice)e(of)j (a)f(linear)g(represen)o(tation)f(for)h(the)g(predictors)g Fo(V)1615 308 y Fm(t)1630 301 y Ft(\()p Fo(i)p Ft(\))g(\(see,)150 410 y(e.g.,)e(Da)o(y)o(an,)h(1992\).)29 b(Learning)20 b(rules)e(for)g(other)h(represen)o(tations)f(can)h(b)q(e)g(obtained)150 519 y(using)c(gradien)o(t)f(descen)o(t)g(but)g(these)h(are)f(not)h (considered)f(here.)20 b(In)14 b(practice)g(TD\()p Fo(\025)p Ft(\))h(is)150 626 y(usually)f(applied)f(to)i(an)f(absorbing)i(c)o (hain)d(th)o(us)h(allo)o(wing)g(the)g(use)g(of)g(either)f(the)h(batc)o (h)150 730 y(or)j(the)f(on-line)f(v)o(ersion)h(but)g(the)g(latter)g(is) g(usually)g(preferred.)150 928 y Fw(Con)n(v)n(ergence)25 b(of)j(TD\()p Fb(\025)p Fw(\))150 1089 y Ft(As)15 b(w)o(e)g(are)h(in)o (terested)e(in)h(strong)h(forms)f(of)g(con)o(v)o(ergence)f(w)o(e)h (need)g(to)h(mo)q(dify)e(the)h(al-)150 1196 y(gorithm)e(sligh)o(tly)l (.)19 b(The)c(learning)f(rate)g(parameters)f Fo(\013)1180 1203 y Fm(n)1218 1196 y Ft(are)h(replaced)g(b)o(y)f Fo(\013)1584 1203 y Fm(n)1608 1196 y Ft(\()p Fo(i)p Ft(\))h(whic)o(h)150 1307 y(satisfy)304 1274 y Fj(P)347 1318 y Fm(n)379 1307 y Fo(\013)410 1314 y Fm(n)434 1307 y Ft(\()p Fo(i)p Ft(\))i(=)g Fv(1)h Ft(and)722 1274 y Fj(P)766 1318 y Fm(n)798 1307 y Fo(\013)829 1289 y Fk(2)829 1319 y Fm(n)852 1307 y Ft(\()p Fo(i)p Ft(\))f Fo(<)g Fv(1)i Ft(uniformly)d(w.p.1.)25 b(These)18 b(parameters)150 1414 y(allo)o(w)g(async)o(hronous)i(up)q (dating)g(and)f(they)f(can,)h(in)f(general,)h(b)q(e)f(random)h(v)m (ariables.)150 1517 y(The)12 b(con)o(v)o(ergence)d(of)j(the)g (algorithm)e(is)h(guaran)o(teed)h(b)o(y)f(the)h(follo)o(wing)f(theorem) f(whic)o(h)150 1621 y(is)16 b(an)h(application)f(of)g(Theorem)f(1.)150 1779 y Fg(Theorem)i(3)24 b Fp(F)l(or)d(any)g(\014nite)i(absorbing)g (Markov)e(chain,)i(for)e(any)g(distribution)h(of)150 1883 y(starting)j(states)g(with)g(no)g(inac)n(c)n(essible)h(states,)g (and)f(for)f(any)g(distributions)h(of)f(the)150 1989 y(c)n(osts)17 b(with)h(\014nite)h(varianc)n(es)f(the)g(TD\()p Fo(\025)p Fp(\))f(algorithm)h(given)h(by)203 2081 y(1\))330 2281 y Fo(V)358 2288 y Fm(n)p Fk(+1)427 2281 y Ft(\()p Fo(i)p Ft(\))13 b(=)h Fo(V)575 2288 y Fm(n)599 2281 y Ft(\()p Fo(i)p Ft(\))d(+)g Fo(\013)745 2288 y Fm(n)768 2281 y Ft(\()p Fo(i)p Ft(\))845 2227 y Fm(m)831 2239 y Fj(X)832 2330 y Fm(t)p Fk(=1)891 2281 y Ft([)p Fo(c)926 2288 y Fm(i)938 2292 y Fi(t)965 2281 y Ft(+)g Fo(\015)s(V)1070 2288 y Fm(n)1094 2281 y Ft(\()p Fo(i)1130 2288 y Fm(t)p Fk(+1)1189 2281 y Ft(\))g Fv(\000)g Fo(V)1297 2288 y Fm(n)1321 2281 y Ft(\()p Fo(i)1357 2288 y Fm(t)1371 2281 y Ft(\)])1438 2227 y Fm(t)1414 2239 y Fj(X)1412 2332 y Fm(k)q Fk(=1)1476 2281 y Ft(\()p Fo(\015)s(\025)p Ft(\))1570 2260 y Fm(t)p Fl(\000)p Fm(k)1632 2281 y Fo(\037)1663 2288 y Fm(i)1677 2281 y Ft(\()p Fo(k)r Ft(\))272 2451 y Fj(P)316 2495 y Fm(n)348 2485 y Fo(\013)379 2492 y Fm(n)402 2485 y Ft(\()p Fo(i)p Ft(\))i(=)h Fv(1)j Fp(and)684 2451 y Fj(P)728 2495 y Fm(n)760 2485 y Fo(\013)791 2467 y Fk(2)791 2497 y Fm(n)814 2485 y Ft(\()p Fo(i)p Ft(\))d Fo(<)f Fv(1)18 b Fp(uniformly)f(over)h Fo(i)f Fp(w.p.1.)951 2775 y Ft(17)p eop %%Page: 18 18 18 17 bop 203 192 a Fp(2\))386 392 y Fo(V)414 399 y Fm(t)p Fk(+1)474 392 y Ft(\()p Fo(i)p Ft(\))13 b(=)h Fo(V)622 399 y Fm(t)637 392 y Ft(\()p Fo(i)p Ft(\))d(+)g Fo(\013)783 399 y Fm(t)798 392 y Ft(\()p Fo(i)p Ft(\)[)p Fo(c)888 399 y Fm(i)900 403 y Fi(t)926 392 y Ft(+)g Fo(\015)s(V)1031 399 y Fm(t)1046 392 y Ft(\()p Fo(i)1082 399 y Fm(t)p Fk(+1)1142 392 y Ft(\))g Fv(\000)g Fo(V)1250 399 y Fm(t)1265 392 y Ft(\()p Fo(i)1301 399 y Fm(t)1316 392 y Ft(\)])1382 338 y Fm(t)1359 350 y Fj(X)1356 443 y Fm(k)q Fk(=1)1421 392 y Ft(\()p Fo(\015)s(\025)p Ft(\))1515 371 y Fm(t)p Fl(\000)p Fm(k)1577 392 y Fo(\037)1608 399 y Fm(i)1621 392 y Ft(\()p Fo(k)r Ft(\))272 562 y Fj(P)316 606 y Fm(t)339 595 y Fo(\013)370 602 y Fm(t)385 595 y Ft(\()p Fo(i)p Ft(\))16 b(=)h Fv(1)i Fp(and)676 562 y Fj(P)720 606 y Fm(n)752 595 y Fo(\013)783 577 y Fk(2)783 608 y Fm(t)803 595 y Ft(\()p Fo(i)p Ft(\))d Fo(<)h Fv(1)i Fp(uniformly)g(w.p.1)g(and)h (within)f(se)n(quenc)n(es)272 705 y Fo(\013)303 712 y Fm(t)318 705 y Ft(\()p Fo(i)p Ft(\))p Fo(=max)494 712 y Fm(t)p Fl(2)p Fm(S)554 705 y Fo(\013)585 712 y Fm(t)600 705 y Ft(\()p Fo(i)p Ft(\))14 b Fv(!)f Ft(1)18 b Fp(uniformly)f(over)h Fo(i)f Fp(w.p.1.)150 856 y(c)n(onver)n(ges)h(to)g(the)g(optimal)f(pr)n (e)n(dictions)g(w.p.1)h(pr)n(ovide)n(d)e Fo(\015)s(;)8 b(\025)14 b Fv(2)g Ft([0)p Fo(;)8 b Ft(1])18 b Fp(with)f Fo(\015)s(\025)e(<)f Ft(1)p Fp(.)248 1019 y Fg(Pro)r(of)25 b(for)h(\(1\):)34 b Ft(Using)22 b(the)h(ideas)f(describ)q(ed)g(in)g (the)h(previous)f(section)g(the)150 1125 y(learning)16 b(rule)g(can)g(b)q(e)g(written)g(as)450 1316 y Fo(V)478 1323 y Fm(n)p Fk(+1)547 1316 y Ft(\()p Fo(i)p Ft(\))41 b(=)h Fo(V)751 1323 y Fm(n)775 1316 y Ft(\()p Fo(i)p Ft(\))10 b(+)h Fo(\013)920 1323 y Fm(n)944 1316 y Ft(\()p Fo(i)p Ft(\)[)p Fo(G)1051 1323 y Fm(n)1074 1316 y Ft(\()p Fo(i)p Ft(\))g Fv(\000)1236 1282 y Fo(m)p Ft(\()p Fo(i)p Ft(\))p 1194 1304 181 2 v 1194 1350 a(E)q Fv(f)p Fo(m)p Ft(\()p Fo(i)p Ft(\))p Fv(g)1380 1316 y Fo(V)1408 1323 y Fm(n)1432 1316 y Ft(\()p Fo(i)p Ft(\)])485 1522 y Fo(G)523 1529 y Fm(n)547 1522 y Ft(\()p Fo(i)p Ft(\))41 b(=)806 1488 y(1)p 728 1510 V 728 1556 a(E)p Fv(f)p Fo(m)p Ft(\()p Fo(i)p Ft(\))p Fv(g)921 1463 y Fm(m)p Fk(\()p Fm(i)p Fk(\))927 1480 y Fj(X)925 1572 y Fm(k)q Fk(=1)1000 1522 y Fo(V)1040 1501 y Fm(\025)1028 1534 y(n)1062 1522 y Ft(\()p Fo(i)p Ft(;)8 b Fo(k)r Ft(\))150 1727 y(where)19 b Fo(V)334 1709 y Fm(\025)322 1739 y(n)356 1727 y Ft(\()p Fo(i)p Ft(;)8 b Fo(k)r Ft(\))20 b(is)f(an)i(estimate)d(calculated)h(at) h(the)f Fo(k)1214 1709 y Fm(th)1269 1727 y Ft(o)q(ccurence)g(of)i (state)f Fo(i)f Ft(in)h(a)150 1834 y(sequence)14 b(and)i(for)g (mathematical)c(con)o(v)o(enience)h(w)o(e)h(ha)o(v)o(e)h(made)f(the)h (transformation)150 1940 y Fo(\013)181 1947 y Fm(n)205 1940 y Ft(\()p Fo(i)p Ft(\))f Fv(!)g Ft(E)p Fv(f)p Fo(m)p Ft(\()p Fo(i)p Ft(\))p Fv(g)p Fo(\013)550 1947 y Fm(n)573 1940 y Ft(\()p Fo(i)p Ft(\),)i(where)g Fo(m)p Ft(\()p Fo(i)p Ft(\))h(is)f(the)h(n)o(um)o(b)q(er)e(of)i(times)d(state)k Fo(i)e Ft(w)o(as)h(visited)150 2047 y(during)f(the)g(sequence.)248 2153 y(T)l(o)c(apply)f(Theorem)g(1)h(w)o(e)f(subtract)h Fo(V)977 2135 y Fl(\003)996 2153 y Ft(\()p Fo(i)p Ft(\),)g(the)g (optimal)e(predictions,)h(from)g(b)q(oth)150 2263 y(sides)21 b(of)g(the)f(learning)h(equation.)35 b(By)20 b(iden)o(tifying)f Fo(\013)1203 2270 y Fm(n)1226 2263 y Ft(\()p Fo(i)p Ft(\))i(:=)h Fo(\013)1407 2270 y Fm(n)1430 2263 y Ft(\()p Fo(i)p Ft(\))p Fo(m)p Ft(\()p Fo(i)p Ft(\))p Fo(=)p Ft(E)o Fv(f)p Fo(m)p Ft(\()p Fo(i)p Ft(\))p Fv(g)p Ft(,)150 2372 y Fo(\014)178 2379 y Fm(n)201 2372 y Ft(\()p Fo(i)p Ft(\))13 b(:=)h Fo(\013)366 2379 y Fm(n)389 2372 y Ft(\()p Fo(i)p Ft(\),)g(and)h Fo(F)597 2379 y Fm(n)620 2372 y Ft(\()p Fo(i)p Ft(\))e(:=)h Fo(G)792 2379 y Fm(n)816 2372 y Ft(\()p Fo(i)p Ft(\))7 b Fv(\000)g Fo(V)962 2354 y Fl(\003)982 2372 y Ft(\()p Fo(i)p Ft(\))p Fo(m)p Ft(\()p Fo(i)p Ft(\))p Fo(=)p Ft(E)o Fv(f)p Fo(m)p Ft(\()p Fo(i)p Ft(\))p Fv(g)14 b Ft(w)o(e)f(need)h(to)h (sho)o(w)g(that)150 2481 y(these)g(satisfy)g(the)g(conditions)g(of)h (Theorem)e(1.)21 b(F)l(or)15 b Fo(\013)1176 2488 y Fm(n)1200 2481 y Ft(\()p Fo(i)p Ft(\))g(and)g Fo(\014)1391 2488 y Fm(n)1414 2481 y Ft(\()p Fo(i)p Ft(\))g(this)g(is)g(ob)o(vious.)951 2775 y(18)p eop %%Page: 19 19 19 18 bop 150 192 a Ft(W)l(e)14 b(b)q(egin)g(here)g(b)o(y)f(sho)o(wing) i(that)f Fo(F)849 199 y Fm(n)872 192 y Ft(\()p Fo(i)p Ft(\))g(indeed)f(is)h(a)h(con)o(traction)f(mapping.)20 b(T)l(o)14 b(this)150 298 y(end,)223 452 y(max)262 481 y Fm(i)322 452 y Fv(j)p Ft(E)p Fv(f)p Fo(F)426 459 y Fm(n)449 452 y Ft(\()p Fo(i)p Ft(\))f Fv(j)h Fo(V)573 459 y Fm(n)597 452 y Fv(gj)g Ft(=)306 623 y(max)345 652 y Fm(i)405 623 y Fv(j)502 589 y Ft(1)p 424 611 181 2 v 424 657 a(E)p Fv(f)p Fo(m)p Ft(\()p Fo(i)p Ft(\))p Fv(g)609 623 y Ft(E)p Fv(f)p Ft(\()p Fo(V)725 603 y Fm(\025)714 635 y(n)748 623 y Ft(\()p Fo(i)p Ft(;)8 b(1\))j Fv(\000)g Fo(V)949 603 y Fl(\003)969 623 y Ft(\()p Fo(i)p Ft(\)\))f(+)h(\()p Fo(V)1161 603 y Fm(\025)1149 635 y(n)1183 623 y Ft(\()p Fo(i)p Ft(;)d(2\))j Fv(\000)g Fo(V)1384 603 y Fl(\003)1404 623 y Ft(\()p Fo(i)p Ft(\)\))g(+)g Fo(:)d(:)g(:)13 b Fv(j)g Fo(V)1664 630 y Fm(n)1688 623 y Fv(gj)150 812 y Ft(whic)o(h)j(can)g(b)q(e)g(b)q(ounded)i(ab)q(o)o(v)o(e)e(b)o(y)f (using)i(the)f(relation)306 972 y Fv(j)p Ft(E)p Fv(f)p Fo(V)417 952 y Fm(\025)406 984 y(n)440 972 y Ft(\()p Fo(i)p Ft(;)8 b Fo(k)r Ft(\))i Fv(\000)h Fo(V)644 952 y Fl(\003)663 972 y Ft(\()p Fo(i)p Ft(\))j Fv(j)f Fo(V)787 979 y Fm(n)811 972 y Fv(gj)348 1106 y(\024)41 b Ft(E)469 1058 y Fj(n)511 1106 y Fv(j)p Ft(E)p Fv(f)p Fo(V)622 1086 y Fm(\025)611 1119 y(n)645 1106 y Ft(\()p Fo(i)p Ft(;)8 b Fo(k)r Ft(\))i Fv(\000)h Fo(V)848 1086 y Fl(\003)868 1106 y Ft(\()p Fo(i)p Ft(\))i Fv(j)h Fo(m)p Ft(\()p Fo(i)p Ft(\))f Fv(\025)h Fo(k)r(;)8 b(V)1205 1113 y Fm(n)1229 1106 y Fv(gj)p Fo(\022)q Ft(\()p Fo(m)p Ft(\()p Fo(i)p Ft(\))i Fv(\000)h Fo(k)r Ft(\))i Fv(j)h Fo(V)1584 1113 y Fm(n)1608 1058 y Fj(o)348 1247 y Fv(\024)41 b Fo(P)7 b Fv(f)p Fo(m)p Ft(\()p Fo(i)p Ft(\))13 b Fv(\025)h Fo(k)r Fv(gj)p Ft(E)p Fv(f)p Fo(V)818 1226 y Fm(\025)807 1259 y(n)840 1247 y Ft(\()p Fo(i)p Ft(\))d Fv(\000)g Fo(V)995 1226 y Fl(\003)1015 1247 y Ft(\()p Fo(i)p Ft(\))i Fv(j)h Fo(V)1139 1254 y Fm(n)1163 1247 y Fv(gj)348 1369 y(\024)41 b Fo(\015)s(P)7 b Fv(f)p Fo(m)p Ft(\()p Fo(i)p Ft(\))13 b Fv(\025)h Fo(k)r Fv(g)8 b Ft(max)782 1397 y Fm(i)842 1369 y Fv(j)p Fo(V)884 1376 y Fm(n)908 1369 y Ft(\()p Fo(i)p Ft(\))i Fv(\000)h Fo(V)1062 1348 y Fl(\003)1082 1369 y Ft(\()p Fo(i)p Ft(\))p Fv(j)150 1546 y Ft(where)20 b Fo(\022)q Ft(\()p Fo(x)p Ft(\))h(=)g(0)f(if)g Fo(x)h(<)g Ft(0)f(and)h(1)g(otherwise.)34 b(Here)19 b(w)o(e)h(ha)o(v)o(e)f(also)i (used)g(the)f(fact)150 1659 y(that)e Fo(V)296 1641 y Fm(\025)285 1672 y(n)319 1659 y Ft(\()p Fo(i)p Ft(\))f(is)h(a)f(con)o (traction)h(mapping)f(indep)q(enden)o(t)f(of)i(p)q(ossible)g(discoun)o (ting.)25 b(As)150 1735 y Fj(P)194 1779 y Fm(k)223 1769 y Fo(P)7 b Fv(f)p Fo(m)p Ft(\()p Fo(i)p Ft(\))14 b Fv(\025)f Fo(k)r Fv(g)h Ft(=)g(E)p Fv(f)p Fo(m)p Ft(\()p Fo(i)p Ft(\))p Fv(g)h Ft(w)o(e)h(\014nally)g(get)513 1926 y(max)553 1954 y Fm(i)612 1926 y Fv(j)p Ft(E)p Fv(f)p Fo(F)716 1933 y Fm(n)739 1926 y Ft(\()p Fo(i)p Ft(\))e Fv(j)g Fo(V)864 1933 y Fm(n)887 1926 y Fv(gj)g(\024)g Fo(\015)d Ft(max)1068 1954 y Fm(i)1128 1926 y Fv(j)p Fo(V)1170 1933 y Fm(n)1194 1926 y Ft(\()p Fo(i)p Ft(\))f Fv(\000)h Fo(V)1348 1905 y Fl(\003)1368 1926 y Ft(\()p Fo(i)p Ft(\))p Fv(j)248 2103 y Ft(The)16 b(v)m(ariance)g(of)g Fo(F)626 2110 y Fm(n)649 2103 y Ft(\()p Fo(i)p Ft(\))g(can)h(b)q(e)f(seen)g(to)h (b)q(e)f(b)q(ounded)h(b)o(y)772 2265 y(E)p Fv(f)p Fo(m)873 2244 y Fk(4)892 2265 y Fv(g)8 b Ft(max)965 2294 y Fm(i)1024 2265 y Fv(j)p Fo(V)1066 2272 y Fm(n)1090 2265 y Ft(\()p Fo(i)p Ft(\))p Fv(j)1159 2244 y Fk(2)150 2440 y Ft(F)l(or)16 b(an)o(y)g(absorbing)h(Mark)o(o)o(v)e(c)o(hain)g(the)g(con)o(v)o (ergence)f(to)j(the)e(terminal)f(state)i(is)f(geo-)150 2550 y(metric)f(and)i(th)o(us)g(for)h(ev)o(ery)d(\014nite)i Fo(k)r Ft(,)g(E)p Fv(f)p Fo(m)989 2532 y Fm(k)1010 2550 y Fv(g)d(\024)h Fo(C)t Ft(\()p Fo(k)r Ft(\),)h(implying)f(that)i(the)g (v)m(ariance)951 2775 y(19)p eop %%Page: 20 20 20 19 bop 150 192 a Ft(of)17 b Fo(F)238 199 y Fm(n)261 192 y Ft(\()p Fo(i)p Ft(\))g(is)f(within)h(the)f(b)q(ounds)i(of)g (Theorem)d(1.)24 b(As)16 b(Theorem)g(1)h(is)g(no)o(w)g(applicable)150 301 y(w)o(e)k(can)h(conclude)f(that)h(the)f(batc)o(h)g(v)o(ersion)g(of) h(TD\()p Fo(\025)p Ft(\))g(con)o(v)o(erges)f(to)h(the)f(optimal)150 407 y(predictions)16 b(w.p.1.)1235 b Fc(2)248 514 y Fg(Pro)r(of)21 b(for)g(\(2\))d Ft(The)h(pro)q(of)h(for)f(the)f(on-line)g(v)o(ersion)g (is)g(ac)o(hiev)o(ed)f(b)o(y)h(sho)o(wing)150 620 y(that)j(the)g (e\013ect)f(of)h(the)g(on-line)f(up)q(dating)i(v)m(anishes)f(in)g(the)f (limit)e(thereb)o(y)i(forcing)150 724 y(the)c(t)o(w)o(o)g(v)o(ersions)g (to)h(b)q(e)f(equal)g(asymptotically)l(.)j(W)l(e)d(view)g(the)g (on-line)g(v)o(ersion)g(as)h(a)150 828 y(batc)o(h)f(algorithm)g(in)g (whic)o(h)f(the)i(up)q(dates)g(are)f(made)g(after)g(eac)o(h)g(complete) e(sequence)150 932 y(but)i(are)h(made)e(in)h(suc)o(h)g(a)g(manner)f(so) i(as)g(to)g(b)q(e)f(equal)g(to)h(those)f(made)f(on-line.)248 1045 y(De\014ne)21 b Fo(G)442 1015 y Fa(0)442 1057 y Fm(n)466 1045 y Ft(\()p Fo(i)p Ft(\))h(=)h Fo(G)642 1052 y Fm(n)666 1045 y Ft(\()p Fo(i)p Ft(\))14 b(+)h Fo(G)826 1027 y Fk(\001)826 1057 y Fm(n)858 1045 y Ft(\()p Fo(i)p Ft(\))21 b(to)h(b)q(e)f(a)h(new)g(batc)o(h)f(estimate)f(taking)h(in)o (to)150 1154 y(accoun)o(t)f(the)h(on-line)f(up)q(dating)h(within)f (sequences.)33 b(Here)19 b Fo(G)1364 1161 y Fm(n)1388 1154 y Ft(\()p Fo(i)p Ft(\))h(is)g(the)h(batc)o(h)f(es-)150 1267 y(timate)g(with)i(the)f(desired)g(prop)q(erties)h(\(see)f(the)h (pro)q(of)h(for)f(\(1\)\))g(and)g Fo(G)1569 1249 y Fk(\001)1569 1279 y Fm(n)1601 1267 y Ft(\()p Fo(i)p Ft(\))g(is)f(the)150 1373 y(di\013erence)f(b)q(et)o(w)o(een)f(the)i(t)o(w)o(o.)34 b(W)l(e)21 b(tak)o(e)f(the)g(new)h(batc)o(h)g(learning)f(parameters)g (to)150 1480 y(b)q(e)d(the)f(maxima)e(o)o(v)o(er)i(a)h(sequence,)f (that)h(is)f Fo(a)1031 1487 y Fm(n)1055 1480 y Ft(\()p Fo(i)p Ft(\))e(=)g(max)1267 1487 y Fm(t)p Fl(2)p Fm(S)1337 1480 y Fo(\013)1368 1487 y Fm(t)1383 1480 y Ft(\()p Fo(i)p Ft(\).)22 b(As)17 b(all)f(the)h Fo(\013)1731 1487 y Fm(t)1745 1480 y Ft(\()p Fo(i)p Ft(\))150 1586 y(satisfy)e(the)g(required)f (conditions)h(uniformly)e(w.p.1)i(these)f(new)h(learning)g(parameters) 150 1690 y(satisfy)h(them)f(as)i(w)o(ell.)248 1794 y(T)l(o)f(c)o (haracterize)f(the)h(new)g(batc)o(h)h(estimate)d(w)o(e)i(consider)g (the)g(on-line)g(up)q(dating)150 1904 y(iteration)11 b(and)i(decomp)q(ose)d(it)i(in)o(to)f(three)g(parallel)g(pro)q(cesses.) 20 b(Let)12 b Fo(V)1452 1886 y Fm(B)1441 1916 y(t)1483 1904 y Ft(\()p Fo(i)p Ft(\))f(b)q(e)h(the)f(v)m(alue)150 2013 y(function)21 b(due)h(to)g(the)f(real)h(batc)o(h)f(up)q(dating)i (with)e Fo(a)1193 2020 y Fm(n)1217 2013 y Ft(\()p Fo(i)p Ft(\))g(as)h(learning)f(parameters,)150 2126 y(let)e Fo(V)264 2108 y Fk(\001)252 2138 y Fm(t)295 2126 y Ft(\()p Fo(i)p Ft(\))h(b)q(e)g(the)g(di\013erence)f(b)q(et)o(w)o(een)g(this)h (batc)o(h)g(v)m(alue)g(function)g(and)g(the)g(v)m(alue)150 2235 y(function)14 b(resulting)f(from)f(the)i(on-line)f(up)q(dating)i (and)g(let)d Fo(R)1290 2242 y Fm(t)1305 2235 y Ft(\()p Fo(i)p Ft(\))i(b)q(e)g(the)f(c)o(hange)h(in)f(the)150 2342 y(v)m(alue)k(function)g(after)g Fo(t)g Ft(on-line)g(up)q(dating)h (steps.)25 b(These)17 b(pro)q(cesses)h(can)f(b)q(e)h(written)951 2775 y(20)p eop %%Page: 21 21 21 20 bop 150 192 a Ft(as)181 320 y Fo(V)221 299 y Fm(B)209 332 y(t)p Fk(+1)270 320 y Ft(\()p Fo(i)p Ft(\))41 b(=)g Fo(V)484 299 y Fm(B)473 332 y(t)515 320 y Ft(\()p Fo(i)p Ft(\))10 b(+)h Fo(a)655 327 y Fm(n)678 320 y Ft(\()p Fo(i)p Ft(\)[)p Fo(c)768 327 y Fm(i)780 331 y Fi(t)807 320 y Ft(+)g Fo(\015)s(V)912 327 y Fm(n)936 320 y Ft(\()p Fo(i)972 327 y Fm(t)p Fk(+1)1031 320 y Ft(\))g Fv(\000)g Fo(V)1139 327 y Fm(n)1163 320 y Ft(\()p Fo(i)1199 327 y Fm(t)1213 320 y Ft(\)])p Fo(\014)1274 327 y Fm(t)1288 320 y Ft(\()p Fo(i)p Ft(\))181 450 y Fo(V)221 430 y Fk(\001)209 463 y Fm(t)p Fk(+1)270 450 y Ft(\()p Fo(i)p Ft(\))41 b(=)g Fo(V)484 430 y Fk(\001)473 463 y Fm(t)516 450 y Ft(\()p Fo(i)p Ft(\))11 b(+)g Fo(\013)662 457 y Fm(t)676 450 y Ft(\()p Fo(i)p Ft(\)[)p Fo(c)766 457 y Fm(i)778 461 y Fi(t)805 450 y Ft(+)g Fo(\015)s(V)910 457 y Fm(n)934 450 y Ft(\()p Fo(i)970 457 y Fm(t)p Fk(+1)1029 450 y Ft(\))g Fv(\000)g Fo(V)1137 457 y Fm(n)1161 450 y Ft(\()p Fo(i)1197 457 y Fm(t)1211 450 y Ft(\))g(+)g Fo(\015)s(R)1355 457 y Fm(t)1370 450 y Ft(\()p Fo(i)1406 457 y Fm(t)p Fk(+1)1466 450 y Ft(\))g Fv(\000)g Fo(R)1583 457 y Fm(t)1598 450 y Ft(\()p Fo(i)1634 457 y Fm(t)1648 450 y Ft(\)])p Fo(\014)1709 457 y Fm(t)1723 450 y Ft(\()p Fo(i)p Ft(\))445 576 y Fv(\000)p Fo(a)510 583 y Fm(n)533 576 y Ft(\()p Fo(i)p Ft(\)[)p Fo(c)623 583 y Fm(i)635 587 y Fi(t)661 576 y Ft(+)g Fo(\015)s(V)766 583 y Fm(n)790 576 y Ft(\()p Fo(i)826 583 y Fm(t)p Fk(+1)886 576 y Ft(\))g Fv(\000)f Fo(V)993 583 y Fm(n)1017 576 y Ft(\()p Fo(i)1053 583 y Fm(t)1068 576 y Ft(\)])p Fo(\014)1129 583 y Fm(t)1143 576 y Ft(\()p Fo(i)p Ft(\))173 697 y Fo(R)210 704 y Fm(t)p Fk(+1)270 697 y Ft(\()p Fo(i)p Ft(\))41 b(=)g Fo(R)482 704 y Fm(t)497 697 y Ft(\()p Fo(i)p Ft(\))11 b(+)g Fo(\013)643 704 y Fm(t)658 697 y Ft(\()p Fo(i)p Ft(\)[)p Fo(c)748 704 y Fm(i)760 708 y Fi(t)786 697 y Ft(+)g Fo(\015)s(V)891 704 y Fm(n)915 697 y Ft(\()p Fo(i)951 704 y Fm(t)p Fk(+1)1010 697 y Ft(\))g Fv(\000)g Fo(V)1118 704 y Fm(n)1142 697 y Ft(\()p Fo(i)1178 704 y Fm(t)1192 697 y Ft(\))g(+)g Fo(\015)s(R)1336 704 y Fm(t)1351 697 y Ft(\()p Fo(i)1387 704 y Fm(t)p Fk(+1)1447 697 y Ft(\))g Fv(\000)g Fo(R)1564 704 y Fm(t)1579 697 y Ft(\()p Fo(i)1615 704 y Fm(t)1629 697 y Ft(\)])p Fo(\014)1690 704 y Fm(t)1704 697 y Ft(\()p Fo(i)p Ft(\))150 835 y(with)24 b(initial)f(conditions)i Fo(V)699 817 y Fm(B)688 848 y Fk(0)729 835 y Ft(\()p Fo(i)p Ft(\))j(=)f Fo(V)905 842 y Fm(n)929 835 y Ft(\()p Fo(i)p Ft(\),)f Fo(V)1063 817 y Fk(\001)1052 848 y(0)1094 835 y Ft(\()p Fo(i)p Ft(\))i(=)f Fo(R)1279 842 y Fk(0)1299 835 y Ft(\()p Fo(i)p Ft(\))g(=)h(0.)46 b(By)24 b(denoting)150 945 y Fo(C)189 927 y Fl(\003)185 957 y Fm(t)222 945 y Ft(=)14 b Fv(j)p Fo(c)309 952 y Fm(i)321 956 y Fi(t)341 945 y Ft(+)t Fo(\015)s(V)451 927 y Fl(\003)471 945 y Ft(\()p Fo(i)507 952 y Fm(t)p Fk(+1)566 945 y Ft(\))t Fv(\000)t Fo(V)672 927 y Fl(\003)692 945 y Ft(\()p Fo(i)728 952 y Fm(t)742 945 y Ft(\))p Fv(j)f Ft(and)h(\001)921 952 y Fm(n)958 945 y Ft(=)p Fv(k)f Fo(V)1062 952 y Fm(n)1091 945 y Fv(\000)t Fo(V)1173 927 y Fl(\003)1207 945 y Fv(k)g Ft(and)g(b)q(ounding)i(the)d(learning)150 1054 y(parameters)j(b)o(y)g (their)g(norms)g(and)h(b)o(y)f(the)g(largest)h(mem)o(b)q(er)d(of)i(the) h(sequence)e(\()p Fo(a)1703 1061 y Fm(n)1726 1054 y Ft(\()p Fo(i)p Ft(\)\))150 1160 y(w)o(e)i(readily)f(obtain)154 1321 y(E)p Fv(fj)p Fo(V)265 1300 y Fk(\001)254 1333 y Fm(t)p Fk(+1)314 1321 y Ft(\()p Fo(i)p Ft(\))p Fv(jg)41 b(\024)g Ft(E)p Fv(fj)p Fo(V)640 1300 y Fk(\001)629 1333 y Fm(t)672 1321 y Ft(\()p Fo(i)p Ft(\))p Fv(jg)10 b Ft(+)889 1287 y(2)p 830 1309 142 2 v 830 1355 a(1)i Fv(\000)f Fo(\015)s(\025)977 1321 y(a)1003 1328 y Fm(n)1026 1321 y Ft(\()p Fo(i)p Ft(\))1089 1260 y Fj(\032)1120 1321 y Fv(k)1165 1287 y Fo(\013)1196 1294 y Fm(t)p 1164 1309 50 2 v 1164 1355 a Fo(a)1190 1362 y Fm(n)1229 1321 y Fv(\000)g Ft(1)j Fv(k)g Ft(\(\001)1416 1328 y Fm(n)1450 1321 y Ft(+)d(E)p Fv(f)p Fo(C)1596 1300 y Fl(\003)1592 1333 y Fm(t)1615 1321 y Fv(g)p Ft(\))g(+)g(E)p Fv(fk)j Fo(R)1853 1328 y Fm(t)1882 1321 y Fv(kg)1932 1260 y Fj(\033)150 1502 y Ft(E)p Fv(fk)g Fo(R)284 1509 y Fm(t)p Fk(+1)358 1502 y Fv(kg)41 b(\024)g Ft(E)p Fv(fk)14 b Fo(R)663 1509 y Fm(t)692 1502 y Fv(kg)d Ft(+)865 1469 y(2)p 807 1491 142 2 v 807 1536 a(1)g Fv(\000)g Fo(\015)s(\025)967 1502 y Fv(k)j Fo(a)1032 1509 y Fm(n)1069 1502 y Fv(k)g Ft(\(\001)1168 1509 y Fm(n)1202 1502 y Ft(+)d(E)p Fv(f)p Fo(C)1348 1482 y Fl(\003)1344 1515 y Fm(t)1367 1502 y Fv(g)g Ft(+)g(E)p Fv(fk)j Fo(R)1586 1509 y Fm(t)1615 1502 y Fv(kg)p Ft(\))p Fo(:)150 1668 y Ft(Let)e(us)h(de\014ne)f Fo(C)17 b Ft(=)d(2)p Fo(=)p Ft(\(1)s Fv(\000)s Fo(\015)s(\025)p Ft(\))e(and)h Fo(C)886 1650 y Fl(\003)919 1668 y Fv(\025)h Ft(E)p Fv(f)p Fo(C)1069 1650 y Fl(\003)1065 1681 y Fm(t)1088 1668 y Fv(g)p Ft(.)20 b(Note)12 b(that)g(suc)o(h)g(a)h Fo(C)1544 1650 y Fl(\003)1575 1668 y Ft(can)f(alw)o(a)o(ys)150 1775 y(b)q(e)18 b(found)h(as)f(the)g(optimal)e(v)m(alue)i(function)g (is)f(b)q(ounded)i(under)f(the)g(assumptions)g(of)150 1879 y(the)e(theorem.)k(By)15 b(iterating)h(w)o(e)g(get)473 2010 y(E)q Fv(fk)d Fo(R)607 2017 y Fm(m)655 2010 y Fv(kg)g(\024)h Fo(C)t Ft(\(\001)870 2017 y Fm(n)903 2010 y Ft(+)d Fo(C)991 1990 y Fl(\003)1011 2010 y Ft(\)[\(1)g(+)g Fo(C)17 b Fv(k)d Fo(a)1264 2017 y Fm(n)1301 2010 y Fv(k)p Ft(\))1345 1990 y Fm(m)1389 2010 y Fv(\000)d Ft(1])150 2148 y(and)20 b(inserting)g(this)f(in)o(to)g(the)h(E)p Fv(fj)p Fo(V)852 2130 y Fk(\001)841 2161 y Fm(t)884 2148 y Ft(\()p Fo(i)p Ft(\))p Fv(jg)f Ft(iteration)g(and)h(noting)g(that)g(it)g(is)f(only)h (a)150 2255 y(sum)15 b(w)o(e)h(get)g(an)h(upp)q(er)g(b)q(ound)173 2411 y(E)p Fv(fj)p Fo(V)284 2391 y Fk(\001)273 2424 y Fm(m)316 2411 y Ft(\()p Fo(i)p Ft(\))p Fv(jg)c(\024)g Fo(a)501 2418 y Fm(n)524 2411 y Ft(\()p Fo(i)p Ft(\))p Fo(mC)661 2391 y Fk(2)680 2411 y Ft(\(\001)740 2418 y Fm(n)774 2411 y Ft(+)e Fo(C)862 2391 y Fl(\003)881 2411 y Ft(\))908 2351 y Fj(\032)940 2411 y Ft(max)979 2439 y Fm(t)1044 2411 y Fv(k)1090 2378 y Fo(\013)1121 2385 y Fm(t)p 1088 2400 50 2 v 1088 2446 a Fo(a)1114 2453 y Fm(n)1153 2411 y Fv(\000)g Ft(1)j Fv(k)g Ft(+\(1)d(+)g Fo(C)18 b Fv(k)13 b Fo(a)1538 2418 y Fm(n)1575 2411 y Fv(k)p Ft(\))1619 2391 y Fm(m)1663 2411 y Fv(\000)e Ft(1)1737 2351 y Fj(\033)1713 2550 y Ft(\(21\))951 2775 y(21)p eop %%Page: 22 22 22 21 bop 248 192 a Ft(As)16 b(the)g(new)h(batc)o(h)g(up)q(dates)g(are) g(made)f(using)h Fo(a)1190 199 y Fm(n)1213 192 y Ft(\()p Fo(i)p Ft(\))f(parameters)g(the)g(quan)o(tit)o(y)150 304 y(of)h(in)o(terest)f(to)h(us)h(is)e(E)q Fv(f)p Ft(E)p Fv(fj)p Fo(V)725 286 y Fk(\001)714 317 y Fm(m)756 304 y Ft(\()p Fo(i)p Ft(\))p Fv(jg)p Fo(=a)900 311 y Fm(n)923 304 y Ft(\()p Fo(i)p Ft(\))p Fv(g)1003 311 y Fm(m)1053 304 y Ft(whic)o(h)g(is)h(a)g(b)q(ound)h(for)f(E)q Fv(f)p Fo(G)1607 286 y Fk(\001)1607 317 y Fm(n)1638 304 y Ft(\()p Fo(i)p Ft(\))p Fv(g)p Ft(.)23 b(In)150 420 y(order)12 b(to)h(use)f(Theorem)f(1)h(w)o(e)g(need)f(to)i(sho)o(w)g(that)f(the)g (estimates)e Fo(G)1426 390 y Fa(0)1426 432 y Fm(n)1463 420 y Ft(ha)o(v)o(e)h(the)h(desired)150 529 y(prop)q(erties.)20 b(In)14 b(other)f(w)o(ords,)h(as)h(the)e Fo(G)908 536 y Fm(n)932 529 y Ft(\()p Fo(i)p Ft(\))g(ha)o(v)o(e)g(these)g(prop)q (erties)g(w)o(e)g(need)h(to)f(sho)o(w)150 642 y(that)k(the)f(e\013ect)f (of)i Fo(G)562 624 y Fk(\001)562 654 y Fm(n)594 642 y Ft(\()p Fo(i)p Ft(\))f(is)g(v)m(anishing.)248 748 y(Let)22 b(us)h(\014rst)g(consider)f(the)h(con)o(traction)f(prop)q(ert)o(y)g(of) h(the)g(estimate.)38 b(In)22 b(an)o(y)150 852 y(absorbing)c(Mark)o(o)o (v)e(c)o(hain)g(the)h(probabilit)o(y)f(of)h(absorption)h(increases)e (geometrically)150 956 y(and)i(th)o(us)f(the)g(probabilit)o(y)g(of)g (ha)o(ving)g Fo(m)g Ft(steps)h(in)f(a)g(sequence)f(can)i(b)q(e)f(b)q (ounded)i(b)o(y)150 1062 y Fo(C)185 1069 y Fm(p)205 1062 y Fo(\015)233 1044 y Fm(m)230 1075 y(p)284 1062 y Ft(where)f Fo(\015)452 1069 y Fm(p)489 1062 y Fo(<)g Ft(1.)27 b(Due)18 b(to)h(the)f(con)o(v)o(ergence)f(of)h Fo(a)1217 1069 y Fm(n)1240 1062 y Ft(\()p Fo(i)p Ft(\),)g(for)h Fo(n)f Ft(su\016cien)o(tly)e(large)150 1178 y(\()p Fo(a)195 1185 y Fm(n)218 1178 y Ft(\()p Fo(i)p Ft(\))j(small)e(enough\),)k(the)e (exp)q(ectation)f(of)i(E)p Fv(fk)f Fo(R)1177 1185 y Fm(m)1229 1178 y Fv(kg)g Ft(o)o(v)o(er)f Fo(m)h Ft(is)g(b)q(ounded)h(and)150 1291 y(go)q(es)h(to)g(zero)f(in)g(the)g(limit)e(w.p.1)i(\(all)g(the)g (terms)f(con)o(tain)h Fo(a)1357 1273 y Fm(k)1357 1303 y(n)1401 1291 y Ft(where)g Fo(k)j Fv(\025)d Ft(1\).)34 b(As)150 1404 y(also)20 b Fv(k)301 1384 y Fm(\013)324 1388 y Fi(t)p 299 1392 40 2 v 299 1421 a Fm(a)318 1425 y Fi(n)357 1404 y Fv(\000)13 b Ft(1)19 b Fv(k)f Ft(con)o(v)o(erges)h (to)g(zero)g(w.p.1,)g(E)p Fv(f)p Fo(G)1128 1386 y Fk(\001)1128 1416 y Fm(n)1160 1404 y Ft(\()p Fo(i)p Ft(\))p Fv(g)f Ft(v)m(anishes)i(in)e(the)h(limit)d(\(see)150 1522 y(Equation)h (\(21\)\).)22 b(Using)16 b(Equation)g(\(21\))h(w)o(e)f(can)h(write)e (these)h(results)g(as)457 1669 y Fv(k)e Ft(E)p Fv(f)p Fo(G)592 1637 y Fa(0)592 1681 y Fm(n)627 1669 y Fv(\000)d Fo(V)716 1648 y Fl(\003)736 1669 y Fv(g)j(k)41 b(\024)h(k)13 b Ft(E)q Fv(f)p Fo(G)1057 1676 y Fm(n)1091 1669 y Fv(\000)e Fo(V)1181 1648 y Fl(\003)1200 1669 y Fv(g)j(k)g Ft(+)g Fv(k)f Fo(G)1406 1648 y Fk(\001)1406 1681 y Fm(n)1452 1669 y Fv(k)841 1799 y(\024)42 b Ft(\()p Fo(\015)969 1767 y Fa(0)993 1799 y Ft(+)11 b Fo(C)1081 1779 y Fk(1)1077 1812 y Fm(n)1100 1799 y Ft(\))j Fv(k)g Fo(V)1200 1806 y Fm(n)1235 1799 y Fv(\000)d Fo(V)1324 1779 y Fl(\003)1357 1799 y Fv(k)j Ft(+)p Fo(C)1473 1779 y Fk(2)1469 1812 y Fm(n)150 1939 y Ft(where)j Fo(C)331 1921 y Fk(1)327 1952 y Fm(n)368 1939 y Ft(and)h Fo(C)503 1921 y Fk(2)499 1952 y Fm(n)540 1939 y Ft(go)g(to)g(zero)f(with)g(w.p.1.)24 b(This)18 b(implies)d(that)i(for)h(an)o(y)f Fo(\017)f(>)g Ft(0)h(and)150 2049 y Fv(k)d Fo(V)217 2056 y Fm(n)252 2049 y Fv(\000)d Fo(V)341 2031 y Fl(\003)374 2049 y Fv(k\035)j Fo(\017)i Ft(there)g(exists)f Fo(\015)i(<)d Ft(1)i(suc)o(h)g(that)632 2195 y Fv(k)e Ft(E)p Fv(f)p Fo(G)767 2163 y Fa(0)767 2208 y Fm(n)802 2195 y Fv(\000)d Fo(V)891 2175 y Fl(\003)911 2195 y Fv(g)i(k\024)h Fo(\015)j Fv(k)c Fo(V)1135 2202 y Fm(n)1170 2195 y Fv(\000)e Fo(V)1260 2175 y Fl(\003)1293 2195 y Fv(k)150 2331 y Ft(for)17 b Fo(n)f Ft(large)g(enough.)248 2437 y(The)g(v)m(ariance)g(of)g(the)g(estimate)e(satis\014es)j (condition)f(\(4\))g(of)g(Theorem)f(1.)22 b(T)l(o)16 b(see)150 2550 y(this,)f(note)h(that)g(in)g(deriving)e(the)i(b)q(ound)h (for)f(E)p Fv(f)p Fo(G)1119 2532 y Fk(\001)1119 2562 y Fm(n)1151 2550 y Ft(\()p Fo(i)p Ft(\))p Fv(g)f Ft(the)g(only)h (random)f(v)m(ariables)951 2775 y(22)p eop %%Page: 23 23 23 22 bop 150 192 a Ft(w)o(ere)19 b Fo(C)305 173 y Fl(\003)301 204 y Fm(t)324 192 y Ft(,)h(the)f(v)m(ariances)h(of)f(whic)o(h)g(are)h (b)q(ounded,)h(and)f Fo(m)p Ft(,)f(the)h(v)m(ariance)f(of)h(whic)o(h) 150 298 y(can)15 b(b)q(e)g(b)q(ounded)g(exactly)f(in)g(the)g(same)g(w)o (a)o(y)g(as)i(the)e(exp)q(ectation;)g(it)h(to)q(o)g(v)m(anishes)g(in) 150 402 y(the)h(limit.)248 496 y(Theorem)c(1)i(no)o(w)g(guaran)o(tees)h (that)f(for)g(an)o(y)f Fo(\017)h Ft(the)f(v)m(alue)h(function)f(in)h (the)f(on-line)150 600 y(algorithm)g(con)o(v)o(erges)h(w.p.1)g(in)o(to) g(some)f Fo(\017)p Ft(-b)q(ounded)i(region)g(of)f Fo(V)1403 581 y Fl(\003)1437 600 y Ft(and)h(therefore)e(the)150 703 y(algorithm)i(itself)g(con)o(v)o(erges)h(to)g Fo(V)808 685 y Fl(\003)844 703 y Ft(w.p.1.)791 b Fc(2)150 897 y Fw(Conclusions)150 1041 y Ft(In)17 b(this)g(pap)q(er)g(w)o(e)g(ha)o (v)o(e)f(extended)g(results)h(from)f(sto)q(c)o(hastic)h(appro)o (ximation)f(theory)150 1145 y(to)g(co)o(v)o(er)e(async)o(hronous)i (relaxation)f(pro)q(cesses)h(whic)o(h)f(ha)o(v)o(e)f(a)i(con)o (traction)f(prop)q(ert)o(y)150 1251 y(with)20 b(resp)q(ect)h(to)g(some) e(maxim)o(um)d(norm)k(\(Theorem)f(1\).)35 b(This)20 b(new)h(class)g(of) g(con-)150 1360 y(v)o(erging)14 b(iterativ)o(e)e(pro)q(cesses)j(is)f (sho)o(wn)h(to)f(include)f(b)q(oth)i(the)f(Q-learning)g(and)h(TD\()p Fo(\025)p Ft(\))150 1467 y(algorithms)20 b(in)h(either)f(their)h (on-line)f(or)i(batc)o(h)f(v)o(ersions.)35 b(W)l(e)21 b(note)g(that)h(the)f(con-)150 1574 y(v)o(ergence)c(of)h(the)f(on-line) h(v)o(ersion)f(of)i(TD\()p Fo(\025)p Ft(\))f(has)h(not)f(b)q(een)g(sho) o(wn)h(previously)l(.)25 b(W)l(e)150 1680 y(also)19 b(wish)f(to)h (emphasize)d(the)i(simplicit)n(y)d(of)k(our)f(results.)27 b(The)18 b(con)o(v)o(ergence)f(pro)q(ofs)150 1787 y(for)j(Q-learning)g (and)g(TD\()p Fo(\025)p Ft(\))h(utilize)d(only)i(high-lev)o(el)e (statistical)h(prop)q(erties)h(of)g(the)150 1893 y(estimates)e(used)h (in)h(these)f(algorithms)f(and)i(do)g(not)g(rely)f(on)h(constructions)g (sp)q(eci\014c)150 1997 y(to)15 b(the)f(algorithms.)20 b(Our)14 b(approac)o(h)i(also)f(sheds)g(additional)f(ligh)o(t)g(on)h (the)f(similarities)150 2103 y(b)q(et)o(w)o(een)h(Q-learning)h(and)h (TD\()p Fo(\025)p Ft(\).)248 2210 y(Although)f(Theorem)e(1)j(is)e (readily)g(applicable)g(to)i(DP-based)g(learning)f(sc)o(hemes,)150 2314 y(the)d(theory)h(of)g(Dynamic)e(Programming)h(is)g(imp)q(ortan)o (t)g(only)g(for)h(its)g(c)o(haracterization)150 2418 y(of)i(the)g(optimal)e(solution)i(and)h(for)f(a)g(con)o(traction)g (prop)q(ert)o(y)g(needed)f(in)g(applying)h(the)150 2521 y(theorem.)j(The)13 b(theorem)f(can)i(b)q(e)f(applied)g(to)h(iterativ)o (e)d(algorithms)i(of)h(di\013eren)o(t)e(t)o(yp)q(es)951 2775 y(23)p eop %%Page: 24 24 24 23 bop 150 192 a Ft(as)17 b(w)o(ell.)248 286 y(Finally)e(w)o(e)h (note)h(that)g(Theorem)e(1)j(can)e(b)q(e)h(extended)f(to)h(co)o(v)o(er) e(pro)q(cesses)j(that)150 389 y(do)c(not)g(sho)o(w)g(the)f(usual)h(con) o(traction)g(prop)q(ert)o(y)f(thereb)o(y)g(increasing)g(its)g (applicabilit)o(y)150 493 y(to)k(algorithms)e(of)i(p)q(ossibly)f(more)f (practical)h(imp)q(ortance.)150 686 y Fw(Pro)r(of)29 b(of)f(Theorem)f(1)150 830 y Ft(In)13 b(this)g(section)g(w)o(e)g(pro)o (vide)g(a)h(detailed)e(pro)q(of)j(of)e(the)g(theorem)f(on)i(whic)o(h)f (the)g(con)o(v)o(er-)150 936 y(gence)18 b(pro)q(ofs)i(for)e(Q-learning) g(and)h(TD\()p Fo(\025)p Ft(\))g(w)o(ere)f(based.)28 b(W)l(e)18 b(in)o(tro)q(duce)g(and)h(pro)o(v)o(e)150 1043 y(three)i(essen)o(tial)g(lemm)o(as,)f(whic)o(h)h(will)g(also)h (help)f(to)h(clarify)f(ties)g(to)h(the)f(literature)150 1147 y(and)e(the)f(ideas)g(b)q(ehind)g(the)f(theorem,)g(follo)o(w)o(ed) g(b)o(y)h(the)g(pro)q(of)h(of)f(Theorem)f(1.)27 b(The)150 1253 y(notation)17 b Fv(k)d(\001)g(k)437 1260 y Fm(W)477 1253 y Ft(=)f(max)619 1260 y Fm(x)649 1253 y Fv(j)e(\001)g Fo(=W)c Ft(\()p Fo(x)p Ft(\))p Fv(j)16 b Ft(will)f(b)q(e)i(used)f(in)g (what)h(follo)o(ws.)150 1408 y Fg(Lemma)f(1)24 b Fp(A)18 b(r)n(andom)e(pr)n(o)n(c)n(ess)523 1560 y Fo(w)558 1567 y Fm(n)p Fk(+1)627 1560 y Ft(\()p Fo(x)p Ft(\))d(=)h(\(1)d Fv(\000)g Fo(\013)893 1567 y Fm(n)917 1560 y Ft(\()p Fo(x)p Ft(\)\))p Fo(w)1037 1567 y Fm(n)1060 1560 y Ft(\()p Fo(x)p Ft(\))f(+)h Fo(\014)1213 1567 y Fm(n)1236 1560 y Ft(\()p Fo(x)p Ft(\))p Fo(r)1324 1567 y Fm(n)1348 1560 y Ft(\()p Fo(x)p Ft(\))p Fo(:)150 1711 y Fp(c)n(onver)n(ges)18 b(to)g(zer)n(o)f(with)g(pr)n(ob)n(ability)g(one)h(if)g(the)g(fol)r (lowing)h(c)n(onditions)f(ar)n(e)f(satis\014e)n(d:)203 1857 y(1\))272 1824 y Fj(P)316 1867 y Fm(n)348 1857 y Fo(\013)379 1864 y Fm(n)402 1857 y Ft(\()p Fo(x)p Ft(\))23 b(=)g Fv(1)p Fp(,)640 1824 y Fj(P)684 1867 y Fm(n)716 1857 y Fo(\013)747 1839 y Fk(2)747 1869 y Fm(n)771 1857 y Ft(\()p Fo(x)p Ft(\))f Fo(<)h Fv(1)p Fp(,)1009 1824 y Fj(P)1053 1867 y Fm(n)1085 1857 y Fo(\014)1113 1864 y Fm(n)1136 1857 y Ft(\()p Fo(x)p Ft(\))f(=)h Fv(1)p Fp(,)h(and)1474 1824 y Fj(P)1517 1867 y Fm(n)1549 1857 y Fo(\014)1580 1839 y Fk(2)1577 1869 y Fm(n)1600 1857 y Ft(\()p Fo(x)p Ft(\))f Fo(<)g Fv(1)272 1964 y Fp(uniformly)17 b(over)h Fo(x)f Fp(w.p.1.)203 2112 y(2\))24 b Ft(E)p Fv(f)p Fo(r)352 2119 y Fm(n)375 2112 y Ft(\()p Fo(x)p Ft(\))p Fv(j)p Fo(P)486 2119 y Fm(n)510 2112 y Fo(;)8 b(\014)560 2119 y Fm(n)583 2112 y Fv(g)14 b Ft(=)f(0)18 b Fp(and)g Ft(E)p Fv(f)p Fo(r)891 2094 y Fk(2)890 2125 y Fm(n)913 2112 y Ft(\()p Fo(x)p Ft(\))p Fv(j)p Fo(P)1024 2119 y Fm(n)1048 2112 y Fo(;)8 b(\014)1098 2119 y Fm(n)1121 2112 y Fv(g)13 b(\024)h Fo(C)21 b Fp(w.p.1,)c(wher)n(e)337 2266 y Fo(P)368 2273 y Fm(n)405 2266 y Ft(=)d Fv(f)p Fo(w)517 2273 y Fm(n)540 2266 y Fo(;)8 b(w)597 2273 y Fm(n)p Fl(\000)p Fk(1)666 2266 y Fo(;)g(:)g(:)g(:)f(;)h(r)797 2273 y Fm(n)p Fl(\000)p Fk(1)865 2266 y Fo(;)g(r)909 2273 y Fm(n)p Fl(\000)p Fk(2)978 2266 y Fo(;)g(:)g(:)g(:)f(;)h(\013) 1118 2273 y Fm(n)p Fl(\000)p Fk(1)1187 2266 y Fo(;)g(\013)1240 2273 y Fm(n)p Fl(\000)p Fk(2)1308 2266 y Fo(;)g(:)g(:)g(:)f(;)h(\014) 1445 2273 y Fm(n)p Fl(\000)p Fk(1)1513 2266 y Fo(;)g(\014)1563 2273 y Fm(n)p Fl(\000)p Fk(2)1631 2266 y Fo(;)g(:)g(:)g(:)p Fv(g)150 2441 y Fp(A)o(l)r(l)24 b(the)f(r)n(andom)e(variables)j(ar)n(e) e(al)r(lowe)n(d)i(to)e(dep)n(end)h(on)g(the)g(p)n(ast)f Fo(P)1504 2448 y Fm(n)1528 2441 y Fp(.)37 b Fo(\013)1611 2448 y Fm(n)1635 2441 y Ft(\()p Fo(x)p Ft(\))22 b Fp(and)150 2550 y Fo(\014)178 2557 y Fm(n)201 2550 y Ft(\()p Fo(x)p Ft(\))17 b Fp(ar)n(e)g(assume)n(d)g(to)g(b)n(e)h(non-ne)n(gative)i(and) e(mutual)r(ly)h(indep)n(endent)g(given)g Fo(P)1688 2557 y Fm(n)1712 2550 y Fp(.)951 2775 y Ft(24)p eop %%Page: 25 25 25 24 bop 248 192 a Fg(Pro)r(of.)19 b Ft(Except)12 b(for)h(the)f(app)q (earance)h(of)g Fo(\014)1063 199 y Fm(n)1086 192 y Ft(\()p Fo(x)p Ft(\))f(this)g(is)g(a)h(standard)h(result.)19 b(With)150 298 y(the)e(ab)q(o)o(v)o(e)h(de\014nitions)f(con)o(v)o (ergence)f(follo)o(ws)i(directly)d(from)i(Dv)o(oretzky's)f(extended)150 405 y(theorem)f(\(Dv)o(oretzky)l(,)f(1956\).)150 556 y Fg(Lemma)i(2)24 b Fp(Consider)17 b(a)h(sto)n(chastic)f(iter)n(ation) 711 695 y Fo(X)751 702 y Fm(n)p Fk(+1)820 695 y Ft(\()p Fo(x)p Ft(\))d(=)g Fo(G)990 702 y Fm(n)1014 695 y Ft(\()p Fo(X)1073 702 y Fm(n)1097 695 y Fo(;)8 b(Y)1147 702 y Fm(n)1170 695 y Fo(;)g(x)p Ft(\))150 846 y Fp(wher)n(e)15 b Fo(G)323 853 y Fm(n)363 846 y Fp(is)g(a)g(se)n(quenc)n(e)i(of)e (functions)h(and)g Fo(Y)1034 853 y Fm(n)1073 846 y Fp(is)f(a)g(r)n (andom)f(pr)n(o)n(c)n(ess.)20 b(L)n(et)15 b Ft(\(\012)p Fo(;)8 b Fv(F)d Fo(;)j(P)f Ft(\))150 953 y Fp(b)n(e)20 b(a)g(pr)n(ob)n(ability)f(sp)n(ac)n(e)g(and)h(assume)g(that)g(the)g(pr) n(o)n(c)n(ess)e(is)i(sc)n(ale)g(invariant,)h(that)f(is,)150 1057 y(w.p.1)e(for)e(al)r(l)j Fo(!)d Fv(2)e Ft(\012)578 1205 y Fo(G)p Ft(\()p Fo(\014)s(X)706 1212 y Fm(n)730 1205 y Fo(;)8 b(Y)780 1212 y Fm(n)804 1205 y Ft(\()p Fo(!)r Ft(\))p Fo(;)g(x)p Ft(\))13 b(=)h Fo(\014)s(G)p Ft(\()p Fo(X)1136 1212 y Fm(n)1160 1205 y Fo(;)8 b(Y)1210 1212 y Fm(n)1233 1205 y Ft(\()p Fo(!)r Ft(\))p Fo(;)g(x)p Ft(\))150 1357 y Fp(Assume)24 b(further)e(that)i(if)e(we)i(kept)g Fv(k)g Fo(X)940 1364 y Fm(n)989 1357 y Fv(k)f Fp(b)n(ounde)n(d)g(by)g (sc)n(aling,)i(then)f Fo(X)1632 1364 y Fm(n)1679 1357 y Fp(would)150 1463 y(c)n(onver)n(ge)17 b(to)g(zer)n(o)f(w.p.1.)22 b(These)17 b(assumptions)f(ar)n(e)g(su\016cient)i(to)e(guar)n(ante)n(e) h(that)f(the)150 1567 y(original)i(pr)n(o)n(c)n(ess)e(c)n(onver)n(ges)i (to)g(zer)n(o)f(w.p.1.)248 1716 y Fg(Pro)r(of.)23 b Ft(Note)16 b(that)i(m)o(ultiply)o(ing)c Fo(X)948 1723 y Fm(n)989 1716 y Ft(b)o(y)j Fo(\014)i Ft(corresp)q(onds)f(to)g(ha)o(ving)e (initialized)150 1819 y(the)23 b(pro)q(cess)g(with)g Fo(\014)s(X)606 1826 y Fk(0)625 1819 y Ft(.)41 b(No)o(w)22 b(\014x)h(some)f(constan)o(t)h Fo(C)t Ft(.)40 b(If)22 b(during)h(the)g(iteration,)150 1926 y Fv(k)g Fo(X)238 1933 y Fm(n)285 1926 y Fv(k)e Ft(increases)g(ab)q(o)o(v)o(e)h Fo(C)t Ft(,)g(then)f Fo(X)915 1933 y Fm(n)961 1926 y Ft(is)g(scaled)g(so)h(that)g Fv(k)h Fo(X)1428 1933 y Fm(n)1475 1926 y Fv(k)p Ft(=)g Fo(C)t Ft(.)36 b(By)21 b(the)150 2032 y(second)d(assumption)g(then)g(this)g(pro)q(cess)g(m)o (ust)f(con)o(v)o(erge)g(w.p.1.)27 b(T)l(o)18 b(sho)o(w)h(that)f(the)150 2139 y(net)k(e\013ect)g(of)h(the)f(corrections)g(m)o(ust)f(sta)o(y)h (\014nite)g(w.p.1)g(w)o(e)g(note)g(that)h(if)f Fv(k)i Fo(X)1727 2146 y Fm(n)1775 2139 y Fv(k)150 2248 y Ft(con)o(v)o(erges)19 b(then)h(for)g(an)o(y)f Fo(\017)h(>)g Ft(0)g(there)f(exists)g Fo(M)1115 2255 y Fm(\017)1152 2248 y Ft(suc)o(h)g(that)i Fv(k)e Fo(X)1459 2255 y Fm(n)1503 2248 y Fv(k)p Fo(<)h(\017)g(<)f(C)24 b Ft(for)150 2355 y(all)16 b Fo(n)e(>)h(M)361 2362 y Fm(\017)394 2355 y Ft(with)h(probabilit)o(y)f(at)i(least)g(1)11 b Fv(\000)g Fo(\017)p Ft(.)22 b(But)16 b(this)g(implies)e(that)j(the)f (iteration)150 2459 y(sta)o(ys)f(b)q(elo)o(w)g Fo(C)j Ft(after)d Fo(M)622 2466 y Fm(\017)653 2459 y Ft(and)g(con)o(v)o(erges) f(to)h(zero)g(without)g(an)o(y)f(further)h(corrections.)150 2550 y Fc(2)951 2775 y Ft(25)p eop %%Page: 26 26 26 25 bop 150 192 a Fg(Lemma)16 b(3)24 b Fp(A)18 b(sto)n(chastic)g(pr)n (o)n(c)n(ess)f Fo(X)883 199 y Fm(n)p Fk(+1)952 192 y Ft(\()p Fo(x)p Ft(\))e(=)f(\(1)e Fv(\000)f Fo(\013)p Ft(\()p Fo(x)p Ft(\)\))p Fo(X)1346 199 y Fm(n)1370 192 y Ft(\()p Fo(x)p Ft(\))g(+)h Fo(\015)s(\014)1553 199 y Fm(n)1576 192 y Ft(\()p Fo(x)p Ft(\))i Fv(k)h Fo(X)1736 199 y Fm(n)1775 192 y Fv(k)150 298 y Fp(c)n(onver)n(ges)j(to)g(zer)n(o) f(w.p.1)g(pr)n(ovide)n(d)203 446 y(1\))24 b Fo(x)13 b Fv(2)i Fo(S)s Fp(,)i(wher)n(e)g Fo(S)k Fp(is)c(a)g(\014nite)i(set.)203 599 y(2\))272 566 y Fj(P)316 609 y Fm(n)348 599 y Fo(\013)379 606 y Fm(n)402 599 y Ft(\()p Fo(x)p Ft(\))i(=)h Fv(1)p Fp(,)636 566 y Fj(P)680 609 y Fm(n)712 599 y Fo(\013)743 581 y Fk(2)743 611 y Fm(n)767 599 y Ft(\()p Fo(x)p Ft(\))f Fo(<)g Fv(1)p Fp(,)1001 566 y Fj(P)1045 609 y Fm(n)1076 599 y Fo(\014)1104 606 y Fm(n)1127 599 y Ft(\()p Fo(x)p Ft(\))h(=)f Fv(1)p Fp(,)1362 566 y Fj(P)1406 609 y Fm(n)1437 599 y Fo(\014)1468 581 y Fk(2)1465 611 y Fm(n)1488 599 y Ft(\()p Fo(x)p Ft(\))h Fo(<)f Fv(1)p Fp(,)i(and)272 708 y Ft(E)p Fv(f)p Fo(\014)358 715 y Fm(n)381 708 y Ft(\()p Fo(x)p Ft(\))p Fv(j)p Fo(P)492 715 y Fm(n)515 708 y Fv(g)14 b(\024)g Ft(E)p Fv(f)p Fo(\013)696 715 y Fm(n)719 708 y Ft(\()p Fo(x)p Ft(\))p Fv(j)p Fo(P)830 715 y Fm(n)854 708 y Fv(g)j Fp(uniformly)g(over)h Fo(x)f Fp(w.p.1.)150 856 y(wher)n(e)426 953 y Fo(P)457 960 y Fm(n)495 953 y Ft(=)d Fv(f)p Fo(X)612 960 y Fm(n)635 953 y Fo(;)8 b(X)697 960 y Fm(n)p Fl(\000)p Fk(1)766 953 y Fo(;)g(:)g(:)g(:)g(;)g(\013)907 960 y Fm(n)p Fl(\000)p Fk(1)975 953 y Fo(;)g(\013)1028 960 y Fm(n)p Fl(\000)p Fk(2)1097 953 y Fo(;)g(:)g(:)g(:)f(;)h(\014)1234 960 y Fm(n)p Fl(\000)p Fk(1)1302 953 y Fo(;)g(\014)1352 960 y Fm(n)p Fl(\000)p Fk(2)1420 953 y Fo(;)g(:)g(:)g(:)o Fv(g)150 1090 y Fo(\013)181 1097 y Fm(n)205 1090 y Ft(\()p Fo(x)p Ft(\))22 b Fp(and)h Fo(\014)421 1097 y Fm(n)444 1090 y Ft(\()p Fo(x)p Ft(\))g Fp(ar)n(e)f(assume)n(d)h(to)g(b)n(e)g (non-ne)n(gative)j(and)d(mutual)r(ly)h(indep)n(endent)150 1196 y(given)19 b Fo(P)308 1203 y Fm(n)332 1196 y Fp(.)248 1353 y Fg(Pro)r(of.)j Ft(Essen)o(tially)16 b(the)g(pro)q(of)i(is)f(an)g (application)g(of)g(Lemma)d(2.)24 b(T)l(o)17 b(this)g(end,)150 1460 y(assume)d(that)i(w)o(e)e(k)o(eep)g Fv(k)g Fo(X)681 1467 y Fm(n)718 1460 y Fv(k\024)g Fo(C)831 1467 y Fk(1)865 1460 y Ft(b)o(y)h(scaling)g(whic)o(h)f(allo)o(ws)h(the)f(iterativ)o(e)f (pro)q(cess)150 1566 y(to)k(b)q(e)f(b)q(ounded)h(b)o(y)511 1723 y Fv(j)p Fo(X)565 1730 y Fm(n)p Fk(+1)633 1723 y Ft(\()p Fo(x)p Ft(\))p Fv(j)d(\024)f Ft(\(1)f Fv(\000)f Fo(\013)915 1730 y Fm(n)938 1723 y Ft(\()p Fo(x)p Ft(\)\))p Fv(j)p Fo(X)1077 1730 y Fm(n)1101 1723 y Ft(\()p Fo(x)p Ft(\))p Fv(j)f Ft(+)h Fo(\015)s(\014)1296 1730 y Fm(n)1319 1723 y Ft(\()p Fo(x)p Ft(\))p Fo(C)1420 1730 y Fk(1)150 1882 y Ft(This)19 b(is)g(linear)f(in)g Fv(j)p Fo(X)567 1889 y Fm(n)591 1882 y Ft(\()p Fo(x)p Ft(\))p Fv(j)g Ft(and)i(can)f(b)q(e)g(easily)f(sho)o(wn)h(to)g(con)o(v)o(erge)f(w.p.1) h(to)g(some)150 1991 y Fo(X)194 1973 y Fl(\003)214 1991 y Ft(\()p Fo(x)p Ft(\),)i(where)f Fv(k)h Fo(X)550 1973 y Fl(\003)591 1991 y Fv(k\024)g Fo(\015)s(C)739 1998 y Fk(1)758 1991 y Ft(.)34 b(Hence,)20 b(for)h(small)e(enough)i Fo(\017)p Ft(,)g(there)f(exists)f Fo(M)1722 1998 y Fk(1)1742 1991 y Ft(\()p Fo(\017)p Ft(\))150 2100 y(suc)o(h)14 b(that)h Fv(k)f Fo(X)441 2107 y Fm(n)479 2100 y Fv(k\024)f Fo(C)591 2107 y Fk(1)611 2100 y Fo(=)p Ft(\(1)8 b(+)g Fo(\017)p Ft(\))14 b(for)h(all)f Fo(n)g(>)f(M)1065 2107 y Fk(1)1085 2100 y Ft(\()p Fo(\017)p Ft(\))h(with)h(probabilit)o(y)e (at)i(least)g Fo(p)1709 2107 y Fk(1)1729 2100 y Ft(\()p Fo(\017)p Ft(\).)150 2210 y(With)20 b(probabilit)o(y)g Fo(p)558 2217 y Fk(1)578 2210 y Ft(\()p Fo(\017)p Ft(\))h(the)f(pro)q (cedure)h(can)g(b)q(e)g(rep)q(eated)g(for)g Fo(C)1459 2217 y Fk(2)1500 2210 y Ft(=)g Fo(C)1594 2217 y Fk(1)1614 2210 y Fo(=)p Ft(\(1)15 b(+)f Fo(\017)p Ft(\).)150 2319 y(Con)o(tin)o(uing)i(in)g(this)h(manner)e(and)i(c)o(ho)q(osing)h Fo(p)1049 2326 y Fm(k)1070 2319 y Ft(\()p Fo(\017)p Ft(\))e(so)i(that) 1311 2286 y Fj(Q)1350 2329 y Fm(k)1379 2319 y Fo(p)1403 2326 y Fm(k)1425 2319 y Ft(\()p Fo(\017)p Ft(\))e(go)q(es)i(to)f(one)f (as)150 2426 y Fo(\017)f Fv(!)h Ft(0)i(w)o(e)f(obtain)g(the)g(w.p.1)h (con)o(v)o(ergence)d(of)j(the)f(b)q(ounded)h(iteration)f(and)h(Lemma) 150 2529 y(2)f(can)f(b)q(e)g(applied.)1248 b Fc(2)951 2775 y Ft(26)p eop %%Page: 27 27 27 26 bop 150 192 a Fg(Theorem)17 b(1)24 b Fp(A)13 b(r)n(andom)e(iter)n (ative)i(pr)n(o)n(c)n(ess)e Ft(\001)1047 199 y Fm(n)p Fk(+1)1115 192 y Ft(\()p Fo(x)p Ft(\))j(=)g(\(1)p Fv(\000)p Fo(\013)1360 199 y Fm(n)1383 192 y Ft(\()p Fo(x)p Ft(\)\)\001)1509 199 y Fm(n)1532 192 y Ft(\()p Fo(x)p Ft(\)+)p Fo(\014)1664 199 y Fm(n)1686 192 y Ft(\()p Fo(x)p Ft(\))p Fo(F)1784 199 y Fm(n)1807 192 y Ft(\()p Fo(x)p Ft(\))150 298 y Fp(c)n(onver)n(ges)k(to)g(zer)n(o)f(w.p.1)g(under)h(the)g(fol)r(lowing) i(assumptions:)203 446 y(1\))k Fo(x)13 b Fv(2)i Fo(S)s Fp(,)i(wher)n(e)g Fo(S)k Fp(is)c(a)g(\014nite)i(set.)203 599 y(2\))272 566 y Fj(P)316 609 y Fm(n)348 599 y Fo(\013)379 606 y Fm(n)402 599 y Ft(\()p Fo(x)p Ft(\))i(=)h Fv(1)p Fp(,)636 566 y Fj(P)680 609 y Fm(n)712 599 y Fo(\013)743 581 y Fk(2)743 611 y Fm(n)767 599 y Ft(\()p Fo(x)p Ft(\))f Fo(<)g Fv(1)p Fp(,)1001 566 y Fj(P)1045 609 y Fm(n)1076 599 y Fo(\014)1104 606 y Fm(n)1127 599 y Ft(\()p Fo(x)p Ft(\))h(=)f Fv(1)p Fp(,)1362 566 y Fj(P)1406 609 y Fm(n)1437 599 y Fo(\014)1468 581 y Fk(2)1465 611 y Fm(n)1488 599 y Ft(\()p Fo(x)p Ft(\))h Fo(<)f Fv(1)p Fp(,)i(and)272 708 y Ft(E)p Fv(f)p Fo(\014)358 715 y Fm(n)381 708 y Ft(\()p Fo(x)p Ft(\))p Fv(j)p Fo(P)492 715 y Fm(n)515 708 y Fv(g)14 b(\024)g Ft(E)p Fv(f)p Fo(\013)696 715 y Fm(n)719 708 y Ft(\()p Fo(x)p Ft(\))p Fv(j)p Fo(P)830 715 y Fm(n)854 708 y Fv(g)p Fp(uniformly)j(over)h Fo(x)f Fp(w.p.1.)203 859 y(3\))24 b Fv(k)14 b Ft(E)p Fv(f)p Fo(F)401 866 y Fm(n)424 859 y Ft(\()p Fo(x)p Ft(\))p Fv(j)p Fo(P)535 866 y Fm(n)558 859 y Fo(;)8 b(\014)608 866 y Fm(n)631 859 y Fv(g)14 b(k)695 866 y Fm(W)735 859 y Fv(\024)g Fo(\015)i Fv(k)e Ft(\001)909 866 y Fm(n)946 859 y Fv(k)971 866 y Fm(W)1011 859 y Fp(,)k(wher)n(e)f Fo(\015)g Fv(2)d Ft(\(0)p Fo(;)8 b Ft(1\))p Fp(.)203 1012 y(4\))24 b Ft(V)l(ar)p Fv(f)p Fo(F)405 1019 y Fm(n)428 1012 y Ft(\()p Fo(x)p Ft(\))p Fv(j)p Fo(P)539 1019 y Fm(n)562 1012 y Fo(;)8 b(\014)612 1019 y Fm(n)635 1012 y Fv(g)14 b(\024)f Fo(C)t Ft(\(1+)h Fv(k)g Ft(\001)940 1019 y Fm(n)977 1012 y Fv(k)1002 1019 y Fm(W)1042 1012 y Ft(\))1061 994 y Fk(2)1081 1012 y Fp(,)j(wher)n(e)h Fo(C)i Fp(is)e(some)f(c)n(onstant.)150 1163 y(Her)n(e)g Fo(P)297 1170 y Fm(n)335 1163 y Ft(=)d Fv(f)p Fo(X)452 1170 y Fm(n)475 1163 y Fo(;)8 b(X)537 1170 y Fm(n)p Fl(\000)p Fk(1)606 1163 y Fo(;)g(:)g(:)g(:)g(;)g(F)748 1170 y Fm(n)p Fl(\000)p Fk(1)816 1163 y Fo(;)g(:)g(:)g(:)f(;)h(\013)956 1170 y Fm(n)p Fl(\000)p Fk(1)1025 1163 y Fo(;)g(:)g(:)g(:)f(;)h(\014) 1162 1170 y Fm(n)p Fl(\000)p Fk(1)1230 1163 y Fo(;)g(:)g(:)g(:)o Fv(g)17 b Fp(stands)g(for)f(the)i(p)n(ast)e(at)150 1272 y(step)i Fo(n)p Fp(.)23 b Fo(F)350 1279 y Fm(n)373 1272 y Ft(\()p Fo(x)p Ft(\))p Fp(,)17 b Fo(\013)502 1279 y Fm(n)525 1272 y Ft(\()p Fo(x)p Ft(\))h Fp(and)f Fo(\014)731 1279 y Fm(n)754 1272 y Ft(\()p Fo(x)p Ft(\))h Fp(ar)n(e)f(al)r(lowe)n (d)h(to)g(dep)n(end)g(on)g(the)g(p)n(ast.)k Fo(\013)1616 1279 y Fm(n)1640 1272 y Ft(\()p Fo(x)p Ft(\))17 b Fp(and)150 1382 y Fo(\014)178 1389 y Fm(n)201 1382 y Ft(\()p Fo(x)p Ft(\))d Fp(ar)n(e)g(assume)n(d)g(to)g(b)n(e)h(non-ne)n(gative)i(and)e (mutual)r(ly)g(indep)n(endent)i(given)f Fo(P)1658 1389 y Fm(n)1681 1382 y Fp(.)22 b(The)150 1491 y(notation)c Fv(k)c(\001)g(k)436 1498 y Fm(W)493 1491 y Fp(r)n(efers)j(to)h(some)f (weighte)n(d)i(maximum)e(norm.)248 1654 y Fg(Pro)r(of.)23 b Ft(By)16 b(de\014ning)h Fo(r)707 1661 y Fm(n)730 1654 y Ft(\()p Fo(x)p Ft(\))e(=)g Fo(F)896 1661 y Fm(n)919 1654 y Ft(\()p Fo(x)p Ft(\))c Fv(\000)g Ft(E)p Fv(f)p Fo(F)1136 1661 y Fm(n)1159 1654 y Ft(\()p Fo(x)p Ft(\))p Fv(j)p Fo(P)1270 1661 y Fm(n)1294 1654 y Fo(;)d(\014)1344 1661 y Fm(n)1366 1654 y Fv(g)17 b Ft(w)o(e)g(can)g(decomp)q(ose)150 1761 y(the)f(iterativ)o(e)e(pro)q(cess)j(in)o(to)f(t)o(w)o(o)g (parallel)g(pro)q(cesses)h(giv)o(en)e(b)o(y)405 1917 y Fo(\016)427 1924 y Fm(n)p Fk(+1)495 1917 y Ft(\()p Fo(x)p Ft(\))41 b(=)h(\(1)11 b Fv(\000)g Fo(\013)817 1924 y Fm(n)841 1917 y Ft(\()p Fo(x)p Ft(\)\))p Fo(\016)948 1924 y Fm(n)970 1917 y Ft(\()p Fo(x)p Ft(\))g(+)g Fo(\014)1124 1924 y Fm(n)1147 1917 y Ft(\()p Fo(x)p Ft(\)E)p Fv(f)p Fo(F)1303 1924 y Fm(n)1326 1917 y Ft(\()p Fo(x)p Ft(\))p Fv(j)p Fo(P)1437 1924 y Fm(n)1460 1917 y Fo(;)d(\014)1510 1924 y Fm(n)1533 1917 y Fv(g)392 2039 y Fo(w)427 2046 y Fm(n)p Fk(+1)495 2039 y Ft(\()p Fo(x)p Ft(\))41 b(=)h(\(1)11 b Fv(\000)g Fo(\013)817 2046 y Fm(n)841 2039 y Ft(\()p Fo(x)p Ft(\)\))p Fo(w)961 2046 y Fm(n)984 2039 y Ft(\()p Fo(x)p Ft(\))f(+)h Fo(\014)1137 2046 y Fm(n)1160 2039 y Ft(\()p Fo(x)p Ft(\))p Fo(r)1248 2046 y Fm(n)1272 2039 y Ft(\()p Fo(x)p Ft(\))375 b(\(22\))150 2198 y(where)24 b(\001)340 2205 y Fm(n)363 2198 y Ft(\()p Fo(x)p Ft(\))i(=)h Fo(\016)542 2205 y Fm(n)565 2198 y Ft(\()p Fo(x)p Ft(\))16 b(+)g Fo(w)736 2205 y Fm(n)759 2198 y Ft(\()p Fo(x)p Ft(\).)44 b(Dividing)23 b(the)h(equations)g(b)o(y)f Fo(W)7 b Ft(\()p Fo(x)p Ft(\))23 b(for)i(eac)o(h)150 2314 y Fo(x)e Ft(and)h(denoting)g Fo(\016)535 2284 y Fa(0)533 2326 y Fm(n)556 2314 y Ft(\()p Fo(x)p Ft(\))h(=)h Fo(\016)733 2321 y Fm(n)756 2314 y Ft(\()p Fo(x)p Ft(\))p Fo(=W)7 b Ft(\()p Fo(x)p Ft(\),)25 b Fo(w)1040 2284 y Fa(0)1039 2326 y Fm(n)1062 2314 y Ft(\()p Fo(x)p Ft(\))h(=)f Fo(w)1252 2321 y Fm(n)1276 2314 y Ft(\()p Fo(x)p Ft(\))p Fo(=W)7 b Ft(\()p Fo(x)p Ft(\),)24 b(and)g Fo(r)1648 2284 y Fa(0)1647 2326 y Fm(n)1670 2314 y Ft(\()p Fo(x)p Ft(\))i(=)150 2429 y Fo(r)172 2436 y Fm(n)195 2429 y Ft(\()p Fo(x)p Ft(\))p Fo(=W)7 b Ft(\()p Fo(x)p Ft(\))20 b(w)o(e)f(can)g(b)q(ound)i (the)e Fo(\016)856 2399 y Fa(0)854 2441 y Fm(n)897 2429 y Ft(pro)q(cess)h(b)o(y)f(assumption)h(3\))g(and)g(rewrite)e(the)951 2775 y(27)p eop %%Page: 28 28 28 27 bop 150 192 a Ft(equation)16 b(pair)g(as)385 357 y Fv(j)p Fo(\016)423 324 y Fa(0)421 369 y Fm(n)p Fk(+1)489 357 y Ft(\()p Fo(x)p Ft(\))p Fv(j)41 b(\024)g Ft(\(1)12 b Fv(\000)f Fo(\013)826 364 y Fm(n)849 357 y Ft(\()p Fo(x)p Ft(\)\))p Fv(j)p Fo(\016)972 324 y Fa(0)970 369 y Fm(n)993 357 y Ft(\()p Fo(x)p Ft(\))p Fv(j)f Ft(+)h Fo(\015)s(\014)1188 364 y Fm(n)1211 357 y Ft(\()p Fo(x)p Ft(\))j Fv(k)f(j)p Fo(\016)1367 324 y Fa(0)1380 357 y Fv(j)e Ft(+)g Fo(w)1490 324 y Fa(0)1489 369 y Fm(n)1526 357 y Fv(k)400 490 y Fo(w)436 458 y Fa(0)435 503 y Fm(n)p Fk(+1)503 490 y Ft(\()p Fo(x)p Ft(\))42 b(=)f(\(1)12 b Fv(\000)f Fo(\013)826 497 y Fm(n)849 490 y Ft(\()p Fo(x)p Ft(\)\))p Fo(w)970 458 y Fa(0)969 503 y Fm(n)992 490 y Ft(\()p Fo(x)p Ft(\))g(+)g Fo(\015)s(\014)1174 497 y Fm(n)1197 490 y Ft(\()p Fo(x)p Ft(\))p Fo(r)1286 458 y Fa(0)1285 503 y Fm(n)1308 490 y Ft(\()p Fo(x)p Ft(\))248 650 y(Assume)21 b(for)j(a)g(momen)o(t)c(that)k(the)f(\001) 1005 657 y Fm(n)1052 650 y Ft(pro)q(cess)h(sta)o(ys)f(b)q(ounded.)44 b(Then)23 b(the)150 763 y(v)m(ariance)c(of)h Fo(r)426 733 y Fa(0)425 775 y Fm(n)448 763 y Ft(\()p Fo(x)p Ft(\))g(is)f(b)q (ounded)h(b)o(y)f(some)g(constan)o(t)h Fo(C)j Ft(and)d(thereb)o(y)e Fo(w)1556 733 y Fa(0)1555 775 y Fm(n)1598 763 y Ft(con)o(v)o(erges)150 870 y(to)g(zero)f(w.p.1)g(according)g(to)h(Lemma)d(1.)25 b(Hence,)15 b(there)i(exists)g Fo(M)22 b Ft(suc)o(h)17 b(that)h(for)g(all)150 982 y Fo(n)i(>)f(M)25 b Fv(k)20 b Fo(w)409 953 y Fa(0)408 995 y Fm(n)451 982 y Fv(k)p Fo(<)f(\017)g Ft(with)h(probabilit)o(y)e(at)i(least)g(1)14 b Fv(\000)f Fo(\017)p Ft(.)31 b(This)20 b(implies)c(that)21 b(the)e Fo(\016)1779 953 y Fa(0)1777 995 y Fm(n)150 1089 y Ft(pro)q(cess)e(can)f(b)q(e)h(further)f(b)q(ounded)h(b)o(y)449 1254 y Fv(j)p Fo(\016)487 1222 y Fa(0)485 1266 y Fm(n)p Fk(+1)553 1254 y Ft(\()p Fo(x)p Ft(\))p Fv(j)c(\024)h Ft(\(1)d Fv(\000)g Fo(\013)834 1261 y Fm(n)857 1254 y Ft(\()p Fo(x)p Ft(\)\))p Fv(j)p Fo(\016)980 1222 y Fa(0)978 1266 y Fm(n)1001 1254 y Ft(\()p Fo(x)p Ft(\))p Fv(j)f Ft(+)h Fo(\015)s(\014)1196 1261 y Fm(n)1219 1254 y Ft(\()p Fo(x)p Ft(\))j Fv(k)g Fo(\016)1362 1222 y Fa(0)1360 1266 y Fm(n)1394 1254 y Ft(+)d Fo(\017)i Fv(k)150 1416 y Ft(with)k (probabilit)o(y)g Fo(>)g Ft(1)12 b Fv(\000)g Fo(\017)p Ft(.)24 b(If)17 b(w)o(e)g(c)o(ho)q(ose)g Fo(C)k Ft(suc)o(h)c(that)h Fo(\015)s Ft(\()p Fo(C)d Ft(+)d(1\))p Fo(=C)20 b(<)c Ft(1)i(then)f(for)150 1532 y Fv(k)d Fo(\016)213 1502 y Fa(0)211 1544 y Fm(n)248 1532 y Fv(k)p Fo(>)f(C)t(\017)633 1650 y(\015)j Fv(k)e Fo(\016)737 1618 y Fa(0)735 1662 y Fm(n)769 1650 y Ft(+)d Fo(\017)j Fv(k\024)f Fo(\015)s Ft(\()p Fo(C)i Ft(+)c(1\))p Fo(=C)18 b Fv(k)c Fo(\016)1258 1618 y Fa(0)1256 1662 y Fm(n)1293 1650 y Fv(k)150 1784 y Ft(and)f(the)e(pro)q(cess)i(de\014ned)e(b)o(y)h(this)g(upp)q(er)g(b)q (ound)h(con)o(v)o(erges)e(to)h(zero)g(w.p.1)g(b)o(y)f(Lemma)150 1896 y(3.)21 b(Th)o(us)13 b Fv(k)h Fo(\016)392 1867 y Fa(0)390 1909 y Fm(n)427 1896 y Fv(k)f Ft(con)o(v)o(erges)f(w.p.1)h(to) g(some)f(v)m(alue)h(b)q(ounded)h(b)o(y)f Fo(C)t(\017)f Ft(whic)o(h)g(guaran)o(tees)150 2003 y(the)k(w.p.1)g(con)o(v)o(ergence) e(of)i(the)g(original)g(pro)q(cess)h(under)f(the)g(b)q(oundedness)h (assump-)150 2105 y(tion.)248 2208 y(By)e(assumption)i(\(4\))f Fo(r)684 2179 y Fa(0)683 2221 y Fm(n)707 2208 y Ft(\()p Fo(x)p Ft(\))g(can)h(b)q(e)f(written)g(as)h(\(1+)e Fv(k)f Fo(\016)1333 2215 y Fm(n)1367 2208 y Ft(+)e Fo(w)1452 2215 y Fm(n)1489 2208 y Fv(k)p Ft(\))p Fo(s)1556 2215 y Fm(n)1580 2208 y Ft(\()p Fo(x)p Ft(\),)j(where)150 2320 y(E)p Fv(f)p Fo(s)231 2302 y Fk(2)231 2332 y Fm(n)255 2320 y Ft(\()p Fo(x)p Ft(\))p Fv(j)p Fo(P)366 2327 y Fm(n)389 2320 y Fv(g)f(\024)f Fo(C)t Ft(.)21 b(Let)16 b(us)h(no)o(w)g(decomp)q(ose)e Fo(w)1086 2327 y Fm(n)1126 2320 y Ft(as)i Fo(u)1214 2327 y Fm(n)1248 2320 y Ft(+)11 b Fo(v)1321 2327 y Fm(n)1360 2320 y Ft(with)338 2488 y Fo(u)366 2495 y Fm(n)p Fk(+1)435 2488 y Ft(\()p Fo(x)p Ft(\))i(=)h(\(1)d Fv(\000)g Fo(\013)701 2495 y Fm(n)725 2488 y Ft(\()p Fo(x)p Ft(\)\))p Fo(u)838 2495 y Fm(n)860 2488 y Ft(\()p Fo(x)p Ft(\))g(+)g Fo(\015)s(\014)1042 2495 y Fm(n)1065 2488 y Ft(\()p Fo(x)p Ft(\))j Fv(k)f Fo(\016)1207 2455 y Fa(0)1205 2500 y Fm(n)1240 2488 y Ft(+)e Fo(u)1317 2495 y Fm(n)1351 2488 y Ft(+)g Fo(v)1424 2495 y Fm(n)1461 2488 y Fv(k)j Fo(s)1523 2495 y Fm(n)1546 2488 y Ft(\()p Fo(x)p Ft(\))951 2775 y(28)p eop %%Page: 29 29 29 28 bop 150 192 a Ft(and)18 b Fo(v)270 199 y Fm(n)310 192 y Ft(con)o(v)o(erges)f(to)g(zero)g(w.p.1)h(b)o(y)e(Lemma)g(1.)24 b(Again)18 b(b)o(y)f(c)o(ho)q(osing)h Fo(C)i Ft(suc)o(h)e(that)150 304 y Fo(\015)s Ft(\()p Fo(C)d Ft(+)d(1\))p Fo(=C)21 b(<)16 b Ft(1)i(w)o(e)f(can)h(b)q(ound)h(the)e Fo(\016)941 275 y Fa(0)939 317 y Fm(n)980 304 y Ft(and)h Fo(u)1104 311 y Fm(n)1145 304 y Ft(pro)q(cesses)h(for)e Fv(k)g Fo(\016)1499 275 y Fa(0)1497 317 y Fm(n)1532 304 y Ft(+)12 b Fo(u)1610 311 y Fm(n)1649 304 y Fv(k)p Fo(>)k(C)t(\017)p Ft(.)150 420 y(The)g(pair)f(\()p Fo(\016)392 390 y Fa(0)390 432 y Fm(n)413 420 y Ft(,)g Fo(u)470 427 y Fm(n)494 420 y Ft(\))g(is)h(then)f(a)h(scale)f(in)o(v)m(arian)o(t)g(pro)q(cess)h (whose)g(b)q(ounded)h(v)o(ersion)e(w)o(as)150 526 y(pro)o(v)o(en)f (earlier)g(to)i(con)o(v)o(erge)e(to)i(zero)f(w.p.1)g(and)h(therefore)e (b)o(y)h(Lemma)e(2)j(it)e(to)q(o)j(con-)150 639 y(v)o(erges)h(to)h (zero)f(w.p.1.)29 b(This)18 b(pro)o(v)o(es)h(the)f(w.p.1)g(con)o(v)o (ergence)f(of)i(the)g(triple)e Fo(\016)1681 609 y Fa(0)1679 652 y Fm(n)1702 639 y Ft(,)i Fo(u)1763 646 y Fm(n)1786 639 y Ft(,)150 746 y(and)e Fo(v)269 753 y Fm(n)308 746 y Ft(b)q(ounding)h(the)e(original)g(pro)q(cess.)812 b Fc(2)781 1000 y Fw(References)150 1146 y Ft(Aoki,)18 b(M.)g(\(1967\).)31 b Fp(Optimization)20 b(of)g(Sto)n(chastic)g (Systems)p Ft(.)29 b(New)19 b(Y)l(ork:)26 b(Academic)248 1252 y(Press.)150 1390 y(Barto,)c(A.)e(G.,)h(Bradtk)o(e,)g(S.)f(J.,)i (&)e(Singh,)i(S.)f(P)l(.)f(\(1993\).)37 b(Learning)21 b(to)h(act)f(using)248 1497 y(real-time)13 b(dynamic)i(programming.)20 b(Submitted)14 b(to:)22 b Fp(AI)17 b(Journal)p Ft(.)150 1645 y(Barto,)g(A.)e(G.,)h(Sutton,)h(R.)f(S.,)g(&)h(W)l(atkins,)f (C.J.C.H.)f(\(1990\).)24 b(Sequen)o(tial)15 b(decision)248 1754 y(problems)10 b(and)j(neural)e(net)o(w)o(orks.)20 b(In)12 b(D.)f(T)l(ouretzky)h(\(Ed.\),)g Fp(A)n(dvanc)n(es)i(in)g(Neur) n(al)248 1861 y(Information)f(Pr)n(o)n(c)n(essing)h(Systems,)h(2)p Ft(,)d(pp.)21 b(686-693.)h(San)13 b(Mateo,)g(CA:)f(Morgan)248 1964 y(Kaufmann.)150 2103 y(Bertsek)m(as,)h(D.)g(P)l(.)g(\(1987\).)22 b Fp(Dynamic)14 b(Pr)n(o)n(gr)n(amming:)20 b(Deterministic)15 b(and)g(Sto)n(chastic)248 2209 y(Mo)n(dels)p Ft(.)21 b(Englew)o(o)q(o)q(d)c(Cli\013s,)f(NJ:)f(Pren)o(tice-Hall.)150 2357 y(Bertsek)m(as,)i(D.)g(P)l(.,)h(&)f(Tsitsiklis,)f(J.)h(N.)g (\(1989\).)27 b Fp(Par)n(al)r(lel)20 b(and)e(Distribute)n(d)h(Compu-) 248 2464 y(tation:)j(Numeric)n(al)c(Metho)n(ds)p Ft(.)j(Englew)o(o)q(o) q(d)d(Cli\013s,)d(NJ:)h(Pren)o(tice-Hall.)951 2775 y(29)p eop %%Page: 30 30 30 29 bop 150 192 a Ft(Da)o(y)o(an,)13 b(P)l(.)g(\(1992\).)22 b(The)13 b(con)o(v)o(ergence)e(of)i(TD\()p Fo(\025)p Ft(\))h(for)f(general)g Fo(\025)p Ft(.)20 b Fp(Machine)15 b(L)n(e)n(arning,)248 296 y(8)p Ft(,)g(341-362.)150 444 y(Da)o(y)o(an,)23 b(P)l(.,)g(&)e(Sejno)o(wski,)i(T.)e(J.)h(\(1993\).)40 b Fp(TD\()p Fo(\025)p Fp(\))22 b(c)n(onver)n(ges)i(with)f(pr)n(ob)n (ability)f(1)p Ft(.)248 550 y(CNL,)16 b(The)g(Salk)g(Institute,)f(San)i (Diego,)f(CA.)150 698 y(Dv)o(oretzky)l(,)f(A.)g(\(1956\).)23 b(On)17 b(sto)q(c)o(hastic)f(appro)o(ximation.)k Fp(Pr)n(o)n(c)n(e)n(e) n(dings)d(of)g(the)h(Thir)n(d)248 805 y(Berkeley)h(Symp)n(osium)d(on)i (Mathematic)n(al)g(Statistics)h(and)e(Pr)n(ob)n(ability)p Ft(.)k(Univ)o(er-)248 909 y(sit)o(y)15 b(of)i(California)f(Press.)150 1057 y(P)o(eng)j(J.,)f(&)h(Williams)d(R.)j(J.)f(\(1993\).)30 b(TD\()p Fo(\025)p Ft(\))20 b(con)o(v)o(erges)e(with)h(probabilit)o(y)f (1.)29 b(De-)248 1163 y(partmen)o(t)14 b(of)j(Computer)e(Science)g (preprin)o(t,)g(Northeastern)h(Univ)o(ersit)o(y)l(.)150 1311 y(Robbins,)g(H.,)f(&)h(Monro,)h(S.)f(\(1951\).)22 b(A)16 b(sto)q(c)o(hastic)h(appro)o(ximation)e(mo)q(del.)20 b Fp(A)o(nnals)248 1418 y(of)d(Mathematic)n(al)h(Statistics,)g(22)p Ft(,)e(400-407.)150 1566 y(Ross,)d(S.)e(M.)f(\(1970\).)22 b Fp(Applie)n(d)13 b(Pr)n(ob)n(ability)f(Mo)n(dels)h(with)g (Optimization)h(Applic)n(ations)p Ft(.)248 1673 y(San)i(F)l(rancisco:) 21 b(Holden-Da)o(y)l(.)150 1821 y(Sutton,)d(R.)f(S.)g(\(1988\).)26 b(Learning)18 b(to)g(predict)f(b)o(y)g(the)g(metho)q(ds)g(of)h(temp)q (oral)e(di\013er-)248 1927 y(ences.)k Fp(Machine)e(L)n(e)n(arning,)f(3) p Ft(,)f(9-44.)150 2075 y(Tsitsiklis)10 b(J.)g(N.)g(\(1993\).)21 b(Async)o(hronous)11 b(sto)q(c)o(hastic)g(appro)o(ximation)f(and)h (Q-learning.)248 2182 y(Submitted)j(to:)22 b Fp(Machine)c(L)n(e)n (arning)p Ft(.)150 2330 y(W)l(atkins,)f(C.J.C.H.)f(\(1989\).)26 b Fp(L)n(e)n(arning)18 b(fr)n(om)f(delaye)n(d)h(r)n(ewar)n(ds)p Ft(.)24 b(PhD)17 b(Thesis,)h(Uni-)248 2436 y(v)o(ersit)o(y)c(of)j(Cam)o (bridge,)d(England.)951 2775 y(30)p eop %%Page: 31 31 31 30 bop 150 192 a Ft(W)l(atkins,)23 b(C.J.C.H,)d(&)i(Da)o(y)o(an,)h (P)l(.)f(\(1992\).)40 b(Q-learning.)f Fp(Machine)23 b(L)n(e)n(arning,)h (8)p Ft(,)248 296 y(279-292.)150 434 y(W)l(erb)q(os,)18 b(P)l(.)f(\(1992\).)27 b(Appro)o(ximate)15 b(dynamic)g(programming)h (for)i(real-time)d(con)o(trol)248 543 y(and)i(neural)f(mo)q(deling.)k (In)d(D.)f(A.)g(White)f(and)i(D.)g(A.)e(Sofge,)i(\(Eds.\),)f Fp(Handb)n(o)n(ok)248 650 y(of)g(Intel)r(ligen)q(t)k(Contr)n(ol:)i (Neur)n(al,)17 b(F)l(uzzy,)g(and)g(A)n(daptive)h(Appr)n(o)n(aches)p Ft(,)c(pp.)22 b(493-)248 754 y(525.)g(New)16 b(Y)l(ork:)k(V)l(an)d (Nostrand)g(Reinhold.)951 2775 y(31)p eop %%Trailer end userdict /end-hook known{end-hook}if %%EOF