(original) (raw)

%!PS-Adobe-2.0 %%Creator: dvips 5.516 Copyright 1986, 1993 Radical Eye Software %%Title: top.dvi %%CreationDate: Fri Jan 19 00:02:01 1996 %%Pages: 30 -1 %%PageOrder: Descend %%BoundingBox: 0 0 612 792 %%EndComments %DVIPSCommandLine: dvips -r -o top.ps top %DVIPSSource: TeX output 1996.01.19:0001 %%BeginProcSet: tex.pro /TeXDict 250 dict def TeXDict begin /N{def}def /B{bind def}N /S{exch}N /X{S N}B /TR{translate}N /isls false N /vsize 11 72 mul N /hsize 8.5 72 mul N /landplus90{false}def /@rigin{isls{[0 landplus90{1 -1}{-1 1} ifelse 0 0 0]concat}if 72 Resolution div 72 VResolution div neg scale isls{landplus90{VResolution 72 div vsize mul 0 exch}{Resolution -72 div hsize mul 0}ifelse TR}if Resolution VResolution vsize -72 div 1 add mul TR matrix currentmatrix dup dup 4 get round 4 exch put dup dup 5 get round 5 exch put setmatrix}N /@landscape{/isls true N}B /@manualfeed{ statusdict /manualfeed true put}B /@copies{/#copies X}B /FMat[1 0 0 -1 0 0]N /FBB[0 0 0 0]N /nn 0 N /IE 0 N /ctr 0 N /df-tail{/nn 8 dict N nn begin /FontType 3 N /FontMatrix fntrx N /FontBBox FBB N string /base X array /BitMaps X /BuildChar{CharBuilder}N /Encoding IE N end dup{/foo setfont}2 array copy cvx N load 0 nn put /ctr 0 N[}B /df{/sf 1 N /fntrx FMat N df-tail}B /dfs{div /sf X /fntrx[sf 0 0 sf neg 0 0]N df-tail}B /E{ pop nn dup definefont setfont}B /ch-width{ch-data dup length 5 sub get} B /ch-height{ch-data dup length 4 sub get}B /ch-xoff{128 ch-data dup length 3 sub get sub}B /ch-yoff{ch-data dup length 2 sub get 127 sub}B /ch-dx{ch-data dup length 1 sub get}B /ch-image{ch-data dup type /stringtype ne{ctr get /ctr ctr 1 add N}if}B /id 0 N /rw 0 N /rc 0 N /gp 0 N /cp 0 N /G 0 N /sf 0 N /CharBuilder{save 3 1 roll S dup /base get 2 index get S /BitMaps get S get /ch-data X pop /ctr 0 N ch-dx 0 ch-xoff ch-yoff ch-height sub ch-xoff ch-width add ch-yoff setcachedevice ch-width ch-height true[1 0 0 -1 -.1 ch-xoff sub ch-yoff .1 add]{ ch-image}imagemask restore}B /D{/cc X dup type /stringtype ne{]}if nn /base get cc ctr put nn /BitMaps get S ctr S sf 1 ne{dup dup length 1 sub dup 2 index S get sf div put}if put /ctr ctr 1 add N}B /I{cc 1 add D }B /bop{userdict /bop-hook known{bop-hook}if /SI save N @rigin 0 0 moveto /V matrix currentmatrix dup 1 get dup mul exch 0 get dup mul add .99 lt{/QV}{/RV}ifelse load def pop pop}N /eop{SI restore showpage userdict /eop-hook known{eop-hook}if}N /@start{userdict /start-hook known{start-hook}if pop /VResolution X /Resolution X 1000 div /DVImag X /IE 256 array N 0 1 255{IE S 1 string dup 0 3 index put cvn put}for 65781.76 div /vsize X 65781.76 div /hsize X}N /p{show}N /RMat[1 0 0 -1 0 0]N /BDot 260 string N /rulex 0 N /ruley 0 N /v{/ruley X /rulex X V}B /V {}B /RV statusdict begin /product where{pop product dup length 7 ge{0 7 getinterval dup(Display)eq exch 0 4 getinterval(NeXT)eq or}{pop false} ifelse}{false}ifelse end{{gsave TR -.1 -.1 TR 1 1 scale rulex ruley false RMat{BDot}imagemask grestore}}{{gsave TR -.1 -.1 TR rulex ruley scale 1 1 false RMat{BDot}imagemask grestore}}ifelse B /QV{gsave transform round exch round exch itransform moveto rulex 0 rlineto 0 ruley neg rlineto rulex neg 0 rlineto fill grestore}B /a{moveto}B /delta 0 N /tail{dup /delta X 0 rmoveto}B /M{S p delta add tail}B /b{S p tail} B /c{-4 M}B /d{-3 M}B /e{-2 M}B /f{-1 M}B /g{0 M}B /h{1 M}B /i{2 M}B /j{ 3 M}B /k{4 M}B /w{0 rmoveto}B /l{p -4 w}B /m{p -3 w}B /n{p -2 w}B /o{p -1 w}B /q{p 1 w}B /r{p 2 w}B /s{p 3 w}B /t{p 4 w}B /x{0 S rmoveto}B /y{ 3 2 roll p a}B /bos{/SS save N}B /eos{SS restore}B end %%EndProcSet %%BeginProcSet: special.pro TeXDict begin /SDict 200 dict N SDict begin /@SpecialDefaults{/hs 612 N /vs 792 N /ho 0 N /vo 0 N /hsc 1 N /vsc 1 N /ang 0 N /CLIP 0 N /rwiSeen false N /rhiSeen false N /letter{}N /note{}N /a4{}N /legal{}N}B /@scaleunit 100 N /@hscale{@scaleunit div /hsc X}B /@vscale{@scaleunit div /vsc X}B /@hsize{/hs X /CLIP 1 N}B /@vsize{/vs X /CLIP 1 N}B /@clip{ /CLIP 2 N}B /@hoffset{/ho X}B /@voffset{/vo X}B /@angle{/ang X}B /@rwi{ 10 div /rwi X /rwiSeen true N}B /@rhi{10 div /rhi X /rhiSeen true N}B /@llx{/llx X}B /@lly{/lly X}B /@urx{/urx X}B /@ury{/ury X}B /magscale true def end /@MacSetUp{userdict /md known{userdict /md get type /dicttype eq{userdict begin md length 10 add md maxlength ge{/md md dup length 20 add dict copy def}if end md begin /letter{}N /note{}N /legal{} N /od{txpose 1 0 mtx defaultmatrix dtransform S atan/pa X newpath clippath mark{transform{itransform moveto}}{transform{itransform lineto} }{6 -2 roll transform 6 -2 roll transform 6 -2 roll transform{ itransform 6 2 roll itransform 6 2 roll itransform 6 2 roll curveto}}{{ closepath}}pathforall newpath counttomark array astore /gc xdf pop ct 39 0 put 10 fz 0 fs 2 F/|______Courier fnt invertflag{PaintBlack}if}N /txpose{pxs pys scale ppr aload pop por{noflips{pop S neg S TR pop 1 -1 scale}if xflip yflip and{pop S neg S TR 180 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{pop S neg S TR pop 180 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{ppr 1 get neg ppr 0 get neg TR}if}{noflips{TR pop pop 270 rotate 1 -1 scale}if xflip yflip and{TR pop pop 90 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{TR pop pop 90 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{TR pop pop 270 rotate ppr 2 get ppr 0 get neg sub neg 0 S TR}if}ifelse scaleby96{ppr aload pop 4 -1 roll add 2 div 3 1 roll add 2 div 2 copy TR .96 dup scale neg S neg S TR}if}N /cp {pop pop showpage pm restore}N end}if}if}N /normalscale{Resolution 72 div VResolution 72 div neg scale magscale{DVImag dup scale}if 0 setgray} N /psfts{S 65781.76 div N}N /startTexFig{/psf$SavedState save N userdict maxlength dict begin /magscale false def normalscale currentpoint TR /psf$ury psfts /psf$urx psfts /psf$lly psfts /psf$llx psfts /psf$y psfts /psf$x psfts currentpoint /psf$cy X /psf$cx X /psf$sx psf$x psf$urx psf$llx sub div N /psf$sy psf$y psf$ury psf$lly sub div N psf$sx psf$sy scale psf$cx psf$sx div psf$llx sub psf$cy psf$sy div psf$ury sub TR /showpage{}N /erasepage{}N /copypage{}N /p 3 def @MacSetUp}N /doclip{ psf$llx psf$lly psf$urx psf$ury currentpoint 6 2 roll newpath 4 copy 4 2 roll moveto 6 -1 roll S lineto S lineto S lineto closepath clip newpath moveto}N /endTexFig{end psf$SavedState restore}N /@beginspecial{SDict begin /SpecialSave save N gsave normalscale currentpoint TR @SpecialDefaults count /ocount X /dcount countdictstack N}N /@setspecial {CLIP 1 eq{newpath 0 0 moveto hs 0 rlineto 0 vs rlineto hs neg 0 rlineto closepath clip}if ho vo TR hsc vsc scale ang rotate rwiSeen{rwi urx llx sub div rhiSeen{rhi ury lly sub div}{dup}ifelse scale llx neg lly neg TR }{rhiSeen{rhi ury lly sub div dup scale llx neg lly neg TR}if}ifelse CLIP 2 eq{newpath llx lly moveto urx lly lineto urx ury lineto llx ury lineto closepath clip}if /showpage{}N /erasepage{}N /copypage{}N newpath }N /@endspecial{count ocount sub{pop}repeat countdictstack dcount sub{ end}repeat grestore SpecialSave restore end}N /@defspecial{SDict begin} N /@fedspecial{end}B /li{lineto}B /rl{rlineto}B /rc{rcurveto}B /np{ /SaveX currentpoint /SaveY X N 1 setlinecap newpath}N /st{stroke SaveX SaveY moveto}N /fil{fill SaveX SaveY moveto}N /ellipse{/endangle X /startangle X /yrad X /xrad X /savematrix matrix currentmatrix N TR xrad yrad scale 0 0 1 startangle endangle arc savematrix setmatrix}N end %%EndProcSet TeXDict begin 40258431 52099146 1000 300 300 (/tmp_mnt/home/u2/jordan/papers/crc/top.dvi) @start /Fa 4 107 df<60F0F07010101020204080040B7D830B>59 D<00FFE0000700000700000700 000E00000E00000E00000E00001C00001C00001C00001C00003800003800003800003800 00700000700000700000700060E000E0E000E0C00081C0004300003E0000131A7C9915> 74 D<03000380030000000000000000000000000000003C004E004E008E008E009C001C 001C0038003800390071007100720072003C00091A7E990D>105 D<0004000E000C000000000000000000000000000000E001300238043804380438007000 700070007000E000E000E000E001C001C001C001C003806380E700C60078000F21809910 >I E /Fb 37 123 df<00800100020004000C0008001800300030003000600060006000 6000E000E000E000E000E000E000E000E000E000E0006000600060006000300030003000 180008000C00040002000100008009267D9B0F>40 D<8000400020001000180008000C00 060006000600030003000300030003800380038003800380038003800380038003800300 0300030003000600060006000C0008001800100020004000800009267E9B0F>I<60F0F0 7010101020204080040B7D830B>44 D<60F0F06004047D830B>46 D<0F8010E02070607870382038007800700070006000C00F8000E000700038003C003CE0 3CE03CC03C4038407030E00F800E187E9713>51 D<30183FF03FE03FC020002000200020 00200027C03860203000380018001C001C401CE01CE01C80184038403030E00F800E187E 9713>53 D<000C0000000C0000000C0000001E0000001E0000003F000000270000002700 000043800000438000004380000081C0000081C0000081C0000100E0000100E00001FFE0 00020070000200700006007800040038000400380008001C0008001C001C001E00FF00FF C01A1A7F991D>65 DI69 D<003F020001C0C60003002E000E001E001C000E001C0006003800060078000200700002 0070000200F0000000F0000000F0000000F0000000F0000000F001FFC070000E0070000E 0078000E0038000E001C000E001C000E000E000E000300160001C06600003F82001A1A7E 991E>71 D<7FFFFF00701C0700401C0100401C0100C01C0180801C0080801C0080801C00 80001C0000001C0000001C0000001C0000001C0000001C0000001C0000001C0000001C00 00001C0000001C0000001C0000001C0000001C0000001C0000001C0000001C000003FFE0 00191A7F991C>84 D<3F8070C070E020700070007007F01C7030707070E070E071E071E0 F171FB1E3C10107E8F13>97 DI<07F80C1C381C30087000 E000E000E000E000E000E0007000300438080C1807E00E107F8F11>I<007E00000E0000 0E00000E00000E00000E00000E00000E00000E00000E0003CE000C3E00380E00300E0070 0E00E00E00E00E00E00E00E00E00E00E00E00E00600E00700E00381E001C2E0007CFC012 1A7F9915>I<07C01C3030187018600CE00CFFFCE000E000E000E0006000300438080C18 07E00E107F8F11>I<01F0031807380E100E000E000E000E000E000E00FFC00E000E000E 000E000E000E000E000E000E000E000E000E000E000E007FE00D1A80990C>I<0FCE1873 30307038703870387038303018602FC02000600070003FF03FFC1FFE600FC003C003C003 C0036006381C07E010187F8F13>II<18003C003C001800 000000000000000000000000FC001C001C001C001C001C001C001C001C001C001C001C00 1C001C001C00FF80091A80990A>I<018003C003C001800000000000000000000000000F C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001 C041C0E180E3007E000A2182990C>IIIII<07E0 1C38300C700E6006E007E007E007E007E007E0076006700E381C1C3807E010107F8F13> II<03C2000C2600381E00300E00700E00E00E00E00E00E00E00E00E00E00E00E0 0E00700E00700E00381E001C2E0007CE00000E00000E00000E00000E00000E00000E0000 7FC012177F8F14>II<1F2060E04020C020C020F0007F003FC01FE000F080708030 C030C020F0408F800C107F8F0F>I<0400040004000C000C001C003C00FFC01C001C001C 001C001C001C001C001C001C201C201C201C201C200E4003800B177F960F>IIIIII<7FF86070407040E041C041C003800700 07000E081C081C08381070107030FFF00D107F8F11>I E /Fc 1 50 df<0C003C00CC000C000C000C000C000C000C000C000C000C000C000C000C00FF8009 107E8F0F>49 D E /Fd 5 108 df<07C01862303260326034C034C038C030403820DA1F 0E0F0B7E8A14>11 D<1E023F046184804800480028003000300030002000200020002000 40004000400F107F8A10>13 D<040C0000000000705898983030606464683006127E910B >105 D<0020002000000000000000000000038004C008C008C000C00180018001800180 03000300030003004600CC0078000B1780910D>I<3C000C000C00180018001800187031 903230340038007F00618061906190C1A0C0C00C117E9010>I E /Fe 2 121 df119 DI E /Ff 15 112 df<0000300000600000C0000180000300000700000E00000C0000180000 380000300000700000E00000C00001C0000180000380000380000300000700000600000E 00000E00000C00001C00001C00001C000018000038000038000038000038000070000070 0000700000700000700000700000700000E00000E00000E00000E00000E00000E00000E0 0000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E0 0000E00000E0000070000070000070000070000070000070000070000038000038000038 00003800001800001C00001C00001C00000C00000E00000E000006000007000003000003 800003800001800001C00000C00000E000007000003000003800001800000C00000E0000 07000003000001800000C0000060000030146377811F>18 DI<0000700001F00003C0000780000E00001C0000380000700000700000F00000 E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E00000E00000E00001C00001C00001C0000380000700000600000E 0000380000700000C000007000003800000E000006000007000003800001C00001C00001 C00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E00000E00000E00000E00000F000007000007000003800001C0000 0E000007800003C00001F000007014637B811F>26 DI<0000180000300000600000E00000C0000180000380000700000600000E00000C0000 1C0000380000380000700000700000E00000E00001E00001C00001C00003800003800003 80000780000700000700000F00000E00000E00001E00001E00001E00001C00001C00003C 00003C00003C00003C000038000078000078000078000078000078000078000078000078 0000700000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F0 0000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F0 0000F00000F00000F0000070000078000078000078000078000078000078000078000078 00003800003C00003C00003C00003C00001C00001C00001E00001E00001E00000E00000E 00000F000007000007000007800003800003800003800001C00001C00001E00000E00000 E000007000007000003800003800001C00000C00000E0000060000070000038000018000 00C00000E0000060000030000018157C768121>32 DI80 D88 DI<000000038000000006600000000C7000 00000CF00000000CF00000001C6000000018000000003800000000380000000038000000 007000000000700000000070000000007000000000F000000000E000000000E000000000 E000000001E000000001E000000001C000000001C000000003C000000003C000000003C0 00000003C000000007800000000780000000078000000007800000000F800000000F8000 00000F000000000F000000001F000000001F000000001F000000001F000000001E000000 003E000000003E000000003E000000003E000000003C000000007C000000007C00000000 7C000000007C000000007800000000F800000000F800000000F800000000F800000000F0 00000001F000000001F000000001F000000001F000000001E000000001E000000003E000 000003E000000003C000000003C000000003C000000003C0000000078000000007800000 0007800000000780000000070000000007000000000F000000000F000000000E00000000 0E000000000E000000001E000000001C000000001C000000001C00000000180000000038 0000000038000000003000000000700000006060000000F060000000F0C0000000E18000 000063000000001E00000000245C7E7F17>I<07C0041FE018307830601FE0800F801605 809D17>101 D104 DI<0000E00003E0000F80 001E00003C0000700000700000E00000E00000E00000E00000E00000E00000E00000E000 00E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E000 00E00000E00000E00001C00001C0000380000700000E00003C0000F00000F000003C0000 0E000007000003800001C00001C00000E00000E00000E00000E00000E00000E00000E000 00E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E000 00E00000E00000E00000E000007000007000003C00001E00000F800003E00000E0134A7C 811C>110 DI E /Fg 2 23 df<01FFFFFFF001FFFFFFF0 01FC0003F000FE0000F000FE000070007F000070007F800070003F800070003FC0006000 1FC00060000FE00020000FE000000007F000000007F000000003F800000003F800000001 F000000001E000000001C000000007000100000E0001800018000300007000030000E000 07000380000E000700001E000C00003C00380001FC007FFFFFFC00FFFFFFF800FFFFFFF8 00241F7C9E28>6 D<00E0070001E00F0003E01F0003E01F0003E01F0003E01F0007C03E 0007C03E0007C03E0007C03E000F807C000F807C000F807C000F807C301F80F8601F80F8 601F81F8C01F8378C03FFE3F803FFC1F003E0000003E0000007C0000007C0000007C0000 007C000000F8000000F8000000700000001C1D7E9320>22 D E /Fh 44 123 df<000FF000007FFC0001F80E0003E01F0007C03F000F803F000F803F000F801E 000F800C000F8000000F8000000F8000000F800000FFFFFF00FFFFFF000F801F000F801F 000F801F000F801F000F801F000F801F000F801F000F801F000F801F000F801F000F801F 000F801F000F801F000F801F000F801F000F801F000F801F000F801F007FF0FFE07FF0FF E01B237FA21F>12 D<387CFEFFFF7F3B03030706060C1C18702008117C8610>44 DI<387CFEFEFE7C3807077C8610>I<00180000780001 F800FFF800FFF80001F80001F80001F80001F80001F80001F80001F80001F80001F80001 F80001F80001F80001F80001F80001F80001F80001F80001F80001F80001F80001F80001 F80001F80001F80001F8007FFFE07FFFE013207C9F1C>49 D<03FC000FFF003C1FC07007 E07C07F0FE03F0FE03F8FE03F8FE01F87C01F83803F80003F80003F00003F00007E00007 C0000F80001F00003E0000380000700000E01801C0180380180700180E00380FFFF01FFF F03FFFF07FFFF0FFFFF0FFFFF015207D9F1C>I<00FE0007FFC00F07E01E03F03F03F03F 81F83F81F83F81F81F03F81F03F00003F00003E00007C0001F8001FE0001FF000007C000 01F00001F80000FC0000FC3C00FE7E00FEFF00FEFF00FEFF00FEFF00FC7E01FC7801F81E 07F00FFFC001FE0017207E9F1C>I<0000E00001E00003E00003E00007E0000FE0001FE0 001FE00037E00077E000E7E001C7E00187E00307E00707E00E07E00C07E01807E03807E0 7007E0E007E0FFFFFEFFFFFE0007E00007E00007E00007E00007E00007E00007E000FFFE 00FFFE17207E9F1C>I<1000201E01E01FFFC01FFF801FFF001FFE001FF8001BC0001800 0018000018000018000019FC001FFF001E0FC01807E01803E00003F00003F00003F80003 F83803F87C03F8FE03F8FE03F8FC03F0FC03F07007E03007C01C1F800FFF0003F8001520 7D9F1C>I<001F8000FFE003F07007C0F00F01F81F01F83E01F83E01F87E00F07C00007C 0000FC0800FC7FC0FCFFE0FD80F0FF00F8FE007CFE007CFC007EFC007EFC007EFC007E7C 007E7C007E7C007E3C007C3E007C1E00F80F00F00783E003FFC000FF0017207E9F1C>I< 6000007800007FFFFE7FFFFE7FFFFC7FFFF87FFFF87FFFF0E00060E000C0C00180C00300 C00300000600000C00001C0000180000380000780000780000F00000F00000F00001F000 01F00001F00003F00003F00003F00003F00003F00003F00003F00001E00017227DA11C> I66 D<0003FE0080001FFF818000FF01E38001F8003F8003E0001F8007C0000F800F80000780 1F800007803F000003803F000003807F000001807E000001807E00000180FE00000000FE 00000000FE00000000FE00000000FE00000000FE00000000FE00000000FE000000007E00 0000007E000001807F000001803F000001803F000003801F800003000F8000030007C000 060003F0000C0001F800380000FF00F000001FFFC0000003FE000021227DA128>II<0003FE0040 001FFFC0C0007F00F1C001F8003FC003F0000FC007C00007C00FC00003C01F800003C03F 000001C03F000001C07F000000C07E000000C07E000000C0FE00000000FE00000000FE00 000000FE00000000FE00000000FE00000000FE00000000FE000FFFFC7E000FFFFC7F0000 1FC07F00001FC03F00001FC03F00001FC01F80001FC00FC0001FC007E0001FC003F0001F C001FC003FC0007F80E7C0001FFFC3C00003FF00C026227DA12C>71 DI76 D78 D<0007FC0000003FFF800000FC07E00003F001F80007E000FC000FC0007E001F80003F00 1F80003F003F00001F803F00001F807F00001FC07E00000FC07E00000FC0FE00000FE0FE 00000FE0FE00000FE0FE00000FE0FE00000FE0FE00000FE0FE00000FE0FE00000FE0FE00 000FE07E00000FC07F00001FC07F00001FC03F00001F803F80003F801F80003F000FC000 7E0007E000FC0003F001F80000FC07E000003FFF80000007FC000023227DA12A>II<07FC00 1FFF803F07C03F03E03F01E03F01F01E01F00001F00001F0003FF003FDF01FC1F03F01F0 7E01F0FC01F0FC01F0FC01F0FC01F07E02F07E0CF81FF87F07E03F18167E951B>97 DI<00FF8007FFE00F83F01F03F03E03F07E03F07C01E07C0000FC0000FC0000FC00 00FC0000FC0000FC00007C00007E00007E00003E00301F00600FC0E007FF8000FE001416 7E9519>I<0001FE000001FE0000003E0000003E0000003E0000003E0000003E0000003E 0000003E0000003E0000003E0000003E0000003E0001FC3E0007FFBE000F81FE001F007E 003E003E007E003E007C003E00FC003E00FC003E00FC003E00FC003E00FC003E00FC003E 00FC003E00FC003E007C003E007C003E003E007E001E00FE000F83BE0007FF3FC001FC3F C01A237EA21F>I<00FE0007FF800F87C01E01E03E01F07C00F07C00F8FC00F8FC00F8FF FFF8FFFFF8FC0000FC0000FC00007C00007C00007E00003E00181F00300FC07003FFC000 FF0015167E951A>I<003F8000FFC001E3E003C7E007C7E00F87E00F83C00F80000F8000 0F80000F80000F80000F8000FFFC00FFFC000F80000F80000F80000F80000F80000F8000 0F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F8000 7FF8007FF80013237FA211>I<03FC1E0FFF7F1F0F8F3E07CF3C03C07C03E07C03E07C03 E07C03E07C03E03C03C03E07C01F0F801FFF0013FC003000003000003800003FFF801FFF F00FFFF81FFFFC3800FC70003EF0001EF0001EF0001EF0001E78003C7C007C3F01F80FFF E001FF0018217E951C>II<1C003F007F007F007F003F001C000000000000000000 000000000000FF00FF001F001F001F001F001F001F001F001F001F001F001F001F001F00 1F001F001F001F001F00FFE0FFE00B247EA310>I107 DIII<00FE0007FFC00F83E01E00F03E00F87C007C7C007C7C00 7CFC007EFC007EFC007EFC007EFC007EFC007EFC007E7C007C7C007C3E00F81F01F00F83 E007FFC000FE0017167E951C>II114 D<0FF3003FFF00781F00600700E00300E00300F00300FC00007FE0007F F8003FFE000FFF0001FF00000F80C00780C00380E00380E00380F00700FC0E00EFFC00C7 F00011167E9516>I<0180000180000180000180000380000380000780000780000F8000 3F8000FFFF00FFFF000F80000F80000F80000F80000F80000F80000F80000F80000F8000 0F80000F80000F81800F81800F81800F81800F81800F830007C30003FE0000F80011207F 9F16>IIIIII<7FFFF07FFFF07C03E07007C0600FC0E01F80C01F00C03E00C07E0000FC0000F80001F0 0003F03007E03007C0300F80701F80703F00603E00E07C03E0FFFFE0FFFFE014167E9519 >I E /Fi 24 123 df<07C0000C3040101040201880601880C00C80C00D00C00E00800E 00800C00C01C00C02C0060C4803F0300120E7E8D17>11 D<0780401FC0403CE080603080 C01080801100000900000A00000A00000A00000C00000C00000C00000800000800000800 0010000010000010000010001214808D12>13 D<0FFF1FFF208040808080018001800100 010003000300030006000600100E7F8D10>28 D<60F0F06004047D830A>58 D<60F0F070101020204040040A7D830A>I<0000300000F00003C0000700001C00007800 01E0000780000E0000380000F00000F000003800000E000007800001E000007800001C00 0007000003C00000F000003014167D921B>I<0008001800300030003000600060006000 C000C000C0018001800180030003000600060006000C000C000C00180018001800300030 003000600060006000C000C0000D217E9812>I<001F8200E04403802C07001C0C001C1C 0008380008300008700008600000E00000E00000E00000C00000C00020C00020C00040E0 00406000806001003002001C1C0007E00017177E9619>67 D<07FFFF8000E0038000E001 0000E0010000E0010001C0010001C0010001C0400001C04000038080000381800003FF80 0003818000070100000701020007010200070004000E0004000E000C000E0008000E0018 001C007000FFFFF00019177F961A>69 D<07FF0000E00000E00000E00000E00001C00001 C00001C00001C0000380000380000380000380000700000700080700080700100E00100E 00300E00200E00601C01E0FFFFC015177F9618>76 D<07F0000FE000F0001E0000B8001E 0000B8002E0000B8004E000138005C000138009C000138011C00011C011C00021C023800 021C043800021C043800021C083800041C107000040E107000040E207000040E40700008 0E40E000080E80E000080F00E000080700E000180601C000FE040FF80023177F9622>I< 07F007F800F000C000B8008000B80080009C0080011C0100011E0100010E0100010E0100 020702000207020002038200020382000401C4000401C4000400E4000400E40008007800 08007800080038000800380018001000FE0010001D177F961C>I<1FFFFE381C0E201C04 601C04401C0440380480380400380000380000700000700000700000700000E00000E000 00E00000E00001C00001C00001C00001C00003C0007FFC0017177F9615>84 D<003E000C000C000C000C0018001800180018073018F0307060706060C060C060C06080 C080C480C4C1C446C838700F177E9612>100 D<1F0006000600060006000C000C000C00 0C0018F01B181C08180838183018301830306030603160616062C022C03C10177E9614> 104 D<0300038003000000000000000000000000001C002400460046008C000C00180018 00180031003100320032001C0009177F960C>I<00180038001000000000000000000000 000001C0022004300430086000600060006000C000C000C000C001800180018001806300 E300C60078000D1D80960E>I<1F0006000600060006000C000C000C000C00181C186618 8E190C32003C003F00318060C060C460C460C8C0C8C0700F177E9612>I<3E0C0C0C0C18 1818183030303060606060C0C8C8C8D07007177E960B>I<383C0044C600470200460200 8E06000C06000C06000C0C00180C00180C40181840181880300880300F00120E7F8D15> 110 D<1C3C22462382230346030603060306030C060C060C0C0C081A3019E01800180030 0030003000FC001014808D12>112 D<030003000600060006000600FFC00C000C000C00 1800180018001800300030803080310031001E000A147F930D>116 D<1C02260646064606860C0C0C0C0C0C0C18181818181818380C7007B000300060706070 C021801E000F147F8D11>121 D<07840FCC187810100020004001800200040008081008 3C3043E081C00E0E7F8D10>I E /Fj 13 117 df<0102040C1818303070606060E0E0E0 E0E0E0E0E0E0E060606070303018180C04020108227D980E>40 D<8040203018180C0C0E 060606070707070707070707070606060E0C0C18183020408008227E980E>I<00300000 3000003000003000003000003000003000003000003000003000003000FFFFFCFFFFFC00 300000300000300000300000300000300000300000300000300000300000300016187E93 1B>43 D<07C018303018701C600C600CE00EE00EE00EE00EE00EE00EE00EE00EE00E600C 600C701C30181C7007C00F157F9412>48 D<03000700FF00070007000700070007000700 070007000700070007000700070007000700070007007FF00C157E9412>I<0F8030E040 708030C038E0384038003800700070006000C00180030006000C08080810183FF07FF0FF F00D157E9412>I61 D76 DI80 D<7FFFF860381840380840380880380480380480380400380000 380000380000380000380000380000380000380000380000380000380000380000380000 380000380007FFC016177F9619>84 D104 D<080008000800180018003800FF80 380038003800380038003800380038403840384038401C800F000A147F930E>116 D E /Fk 14 115 df0 D<70F8F8F87005057C8D0D>I< FFFFFFF07FFFFFF000000000000000000000000000000000000000000000000000000000 FFFFFFF0FFFFFFF000000000000000000000000000000000000000000000000000000000 7FFFFFF0FFFFFFF01C147D9423>17 D<000000C0000003C000000F0000003C000000F000 0003C00000070000001C00000078000001E00000078000001E00000078000000E0000000 780000001E0000000780000001E0000000780000001C0000000700000003C0000000F000 00003C0000000F00000003C0000000C00000000000000000000000000000000000000000 00000000000000007FFFFF80FFFFFFC01A247C9C23>20 D<07C000101FE000103FF80010 783C0030601E0030C0078060C003C1E08001FFC080007F8080003E000000000000000000 0000000000000000000000000000000000000000000000007FFFFFF0FFFFFFF01C147D94 23>39 D<00040000000C0000000C0000000C0000000C0000000C0000000C0000000C0000 000C0000000C0000000C0000000C0000000C0000000C0000000C0000000C0000000C0000 000C0000000C0000000C0000000C0000000C0000000C0000000C0000000C0000000C0000 FFFFFFE0FFFFFFE01B1C7C9B23>63 D<0000FC0007FE001C3E00201E00C01E01801C0380 1C07003C0600380E00381C00601C00003C00003800003800007800007800007000007000 00F00000F00000F00000F00000F00000F00000F80018F800307800707C00C07E00803F83 001FFC0007F0001721809F18>67 D<003FFF0001FFFFE0071E0FF8081E01FC181E007C70 1E003E701E003EC01E001F001C001F001C000F003C000F003C000F003C000F0038000F00 38000E0078000E0078001E0070001C0070001C00F0003800E0003000E0007001E000E001 C0008001C0010003C0060003800800038030000703C0000FFF00001FF00000201F7F9E23 >I<00FC00002003FE0000E0043E0001C0181E0001C0301E000380701E000380F01E0007 00C01C000700001C000E00003C000E00003C001E00003C001C000038001C00003BFFFC00 007FFFF80000780038000070007800007000700000F000700000E000F00000E000F00001 E000E00001E000E00001C001E00001C001E00003C001E000038001E000038001E0000780 01E040070001E1C0060001F3000C0001FE00000000F80023217F9E26>72 D<000F0038006000E001C001C001C001C001C001C001C001C001C001C001C001C001C001 C001C0038007001E00F8001E000700038001C001C001C001C001C001C001C001C001C001 C001C001C001C001C001C000E000600038000F102D7DA117>102 DI106 D<4020C030C030C030C030C030C030C030C030C030C030C030C030C030 C030C030C030C030C030C030C030C030C030C030C030C030C030C030C030C030C030C030 C030C030C030C030C030C030C030C030C030C030C030C030C03040200C2E7BA117>I114 D E /Fl 4 107 df0 D<60F0F06004047D890A>I<060F0F0E1E1E1C 3C383830707060E0C04008117F910A>48 D106 D E /Fm 45 123 df<007C0001C3000701810E01C11E00C11C00E23C00E27800E27800E47800E4F0 00E8F000F0F000F0F000E0F000E07000E07003E030046118383207C01C18147E931D>11 D<007E01C007000E001C003C003800780078007FF0F000F000F000700070007000300018 000C1807E00F147E9312>15 D<1E07C023186023A03043C0304380384380388700700700 700700700700700E00E00E00E00E00E00E00E01C01C01C01C01C01C01C01C03803801803 80000380000380000700000700000700000700000E00000E00000E00000C00151E7E9317 >17 D<001E0000630000C38001C1800381800301C00701C00F01C00E01C01E03C01C03C0 3C03C03C03C03C03C07807807FFF807FFF80780780F00F00F00F00F00F00F00E00F01E00 E01C00E03C00E03800E0300060700060E00070C0003180001E000012207E9F15>I<0180 1801C01C0380380380380380380380380700700700700700700700700E00E00E00E00E00 E00E00E11E01C21E01C21E03C21E05C43F08C439F0783800003800007000007000007000 00700000E00000E00000E00000C00000181E7F931B>22 D<0F00307F00380E00700E0070 0E00700E00E01C00E01C01C01C01C01C0380380300380600380C00381800703000706000 70C000730000FC0000F0000015147E9316>I<0FFFFC1FFFFC3FFFFC608200C084008084 00018400010400010C00030C00030C00020C00060C00060C000E0C000C0E001C0E001C0E 00380F0018060016147E931A>25 D<007FFE01FFFE07FFFE0F07801E03801C01C03801C0 7001C07001C07001C0E00380E00380E00380E00700E00700E00E00600C00301800186000 0F800017147E931A>27 D<0FFFF01FFFF03FFFF0604000C0400080C00000C00000800001 80000180000180000180000380000300000300000700000700000700000E000006000014 147E9314>I<000040000040000080000080000080000080000100000100000100000100 000200000200001FC000E27003841806040C0C040E1C0406380807300807700807700807 E0100EE0100EE0100CE0101C6020387020303020601821C00E470003F800004000004000 00800000800000800000800001000001000001000018297E9F1B>30 D<70F8F8F87005057C840D>58 D<70F8FCFC74040404080810102040060E7C840D>I<00 0100030003000600060006000C000C000C00180018001800300030003000600060006000 C000C000C00180018001800300030003000600060006000C000C000C0018001800180030 0030003000600060006000C000C000C000102D7DA117>61 DI<001F000060C00080 6001003002001803801803C01C07801C03001C00001E00001E00001E003E1E00C11E0380 9C07007C0E007C1C007C3C007C3800787800787800787800F0F000F0F000E0F000E0F001 C0F001C0F00380700300700600380C001C380007E00017227EA018>64 D<000002000000060000000E0000000E0000001E0000001F0000002F0000002F0000004F 0000008F0000008F0000010F0000010F0000020F0000040F0000040F0000080F80000807 80001007800020078000200780007FFF8000400780008007800180078001000780020007 80020007C0040003C00C0003C01E0007C0FF807FFC1E207E9F22>I<00FFFFE0000F0078 000F003C000F001C000F001E001E001E001E001E001E001E001E001E003C003C003C003C 003C0078003C00F0007803C0007FFF80007803C0007801E000F000F000F000F000F000F0 00F0007001E000F001E000F001E000F001E000E003C001E003C003C003C0038003C00F00 07801E00FFFFF0001F1F7E9E22>I<0000FE0200078186001C004C0038003C0060003C00 C0001C01C0001803800018070000180F0000181E0000101E0000103C0000003C00000078 000000780000007800000078000000F0000000F0000000F0000000F0000000F000008070 00008070000080700001003800010038000200180004000C001800060020000381C00000 FE00001F217E9F21>I<00FFFFFF000F000E000F0006000F0002000F0002001E0002001E 0002001E0002001E0002003C0404003C0400003C0400003C0C0000781800007FF8000078 18000078180000F0100000F0100000F0100000F0000401E0000801E0000801E0001001E0 001003C0002003C0006003C0004003C001C0078007C0FFFFFF80201F7E9E22>69 D<00FFF9FFF0000F801F00000F001E00000F001E00000F001E00001E003C00001E003C00 001E003C00001E003C00003C007800003C007800003C007800003C007800007800F00000 7FFFF000007800F000007800F00000F001E00000F001E00000F001E00000F001E00001E0 03C00001E003C00001E003C00001E003C00003C007800003C007800003C007800003C007 800007C00F8000FFF8FFF800241F7E9E26>72 D<001FFF0000F80000F00000F00000F000 01E00001E00001E00001E00003C00003C00003C00003C000078000078000078000078000 0F00000F00000F00000F00001E00001E00301E00781E00F83C00F83C00F0780080700040 E00021C0001F000018207D9E19>74 D<00FF00001FF0000F00003F00000B80003E00000B 80005E00000B80005E0000138000BC00001380013C00001380013C00001380023C000023 800278000023800478000023800878000021C00878000041C010F0000041C020F0000041 C020F0000041C040F0000081C041E0000081C081E0000081C101E0000081C101E0000100 E203C0000100E203C0000100E403C0000100E803C0000200E80780000200F00780000200 F00780000600E00780000F00C00F8000FFE0C1FFF8002C1F7E9E2C>77 D<00FF803FF0000F800780000F800200000BC00200000BC002000013C004000011E00400 0011E004000011E004000020F008000020F008000020F808000020780800004078100000 403C100000403C100000403C100000801E200000801E200000801E200000800F20000100 0F400001000F4000010007C000010007C000020007800002000380000200038000060003 80000F00010000FFE0010000241F7E9E25>I<0001FC0000070700001C01C0003000E000 E0006001C000700380007007800038070000380E0000381E0000381C0000383C0000383C 00003878000078780000787800007878000078F00000F0F00000F0F00000E0F00001E0F0 0001C0F00003C0700003807000070078000F0038001E0038003C001C0070000E00E00007 83800001FC00001D217E9F23>I<00FFFFC0000F0070000F0038000F001C000F001E001E 001E001E001E001E001E001E001E003C003C003C003C003C0078003C0070007800E00078 0380007FFE000078000000F0000000F0000000F0000000F0000001E0000001E0000001E0 000001E0000003C0000003C0000003C0000003C0000007C00000FFFC00001F1F7E9E1D> I<0FFFFFFC1E03C0381803C0181003C0082003C008200780086007800840078008400780 08800F0010000F0000000F0000000F0000001E0000001E0000001E0000001E0000003C00 00003C0000003C0000003C00000078000000780000007800000078000000F0000000F000 0000F0000000F0000001F000007FFFC0001E1F7F9E1B>84 D87 D<00FFF83FF8000FC00F80000F80060000078004000007C008000003 C010000003C020000003E040000001E080000001F100000000F300000000F600000000FC 0000000078000000007C000000007C000000007C00000000BE000000011E000000021E00 0000061F0000000C0F000000080F800000100780000020078000004007C000008003C000 010003E000030003E0000F0007E000FFE01FFE00251F7F9E26>I<00F1800389C0070780 0E03801C03803C0380380700780700780700780700F00E00F00E00F00E00F00E10F01C20 F01C20703C20705C40308C400F078014147E9318>97 D<07803F8007000700070007000E 000E000E000E001C001C001CF01D0C3A0E3C0E380F380F700F700F700F700FE01EE01EE0 1EE01CE03CE038607060E031C01F0010207E9F14>I<007C01C207010E0F1E0F1C0E3C04 780078007800F000F000F000F000F00070017002300418380FC010147E9314>I<000078 0003F80000700000700000700000700000E00000E00000E00000E00001C00001C000F1C0 0389C00707800E03801C03803C0380380700780700780700780700F00E00F00E00F00E00 F00E10F01C20F01C20703C20705C40308C400F078015207E9F18>I<007C01C207010E01 1C013C013802780C7BF07C00F000F000F000F0007000700170023004183807C010147E93 15>I<00007C0000CE00019E00039E00030C000700000700000700000700000E00000E00 000E0000FFF0000E00000E00001C00001C00001C00001C00001C00003800003800003800 00380000380000700000700000700000700000700000E00000E00000E00000E00000C000 01C000318000798000F300006200003C000017297E9F16>I<001E3000713800E0F001C0 700380700780700700E00F00E00F00E00F00E01E01C01E01C01E01C01E01C01E03801E03 800E07800E0B8006170001E700000700000700000E00000E00300E00781C00F038006070 003FC000151D809316>I<01E0000FE00001C00001C00001C00001C00003800003800003 8000038000070000070000071F000761800E80C00F00C00E00E00E00E01C01C01C01C01C 01C01C01C0380380380380380380380704700708700E08700E10700610E006206003C016 207E9F1A>I<00E001E001E000C000000000000000000000000000000E00130023804380 438043808700070007000E000E001C001C001C20384038403840388019000E000B1F7E9E 10>I<0000C00001E00001E00001C0000000000000000000000000000000000000000000 001E00006300004380008380010380010380020700000700000700000700000E00000E00 000E00000E00001C00001C00001C00001C00003800003800003800003800007000007000 30700078E000F1C0006380003E00001328819E13>I<1E07802318C023A06043C0704380 704380708700E00700E00700E00700E00E01C00E01C00E01C00E03821C03841C07041C07 081C03083803101801E017147E931B>110 D<03C1E004621804741C08781C08701E0870 1E10E01E00E01E00E01E00E01E01C03C01C03C01C03C01C0380380780380700380E003C1 C0072380071E000700000700000E00000E00000E00000E00001C00001C0000FFC000171D 819317>112 D<00C000E001C001C001C001C003800380FFF8038007000700070007000E 000E000E000E001C001C001C001C10382038203820384018800F000D1C7F9B10>116 D<0F006060118070F02180E0F821C0E07841C0E0384380E0188381C0100701C0100701C0 100701C0100E0380200E0380200E0380200E0380400E0380400E0380800E078080060781 000709860001F078001D147E9321>119 D<03C1C00C62201034701038F02038F0203860 40700000700000700000700000E00000E00000E00000E02061C040F1C040F1C080E2C080 446300383C0014147E931A>I<0F00601180702180E021C0E041C0E04380E08381C00701 C00701C00701C00E03800E03800E03800E03800E07000C07000C07000E0F00061E0003EE 00000E00000E00001C0078180078380070700060600021C0001F0000141D7E9316>I<01 E02003F04007F8C00C1F8008010000020000040000080000100000600000C00001000002 00000400800801001003003F060061FC0040F80080700013147E9315>I E /Fn 34 123 df<000FE000007FF80000F81C0001E07C0003E07C0007C07C0007C07C00 07C0380007C0000007C0000007C0000007C1FE00FFFFFE00FFFFFE0007C03E0007C03E00 07C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E00 07C03E0007C03E0007C03E0007C03E0007C03E003FF9FFC03FF9FFC01A20809F1D>12 D66 D<0007FC02003FFF0E00FE03 DE03F000FE07E0003E0FC0001E1F80001E3F00000E3F00000E7F0000067E0000067E0000 06FE000000FE000000FE000000FE000000FE000000FE000000FE0000007E0000007E0000 067F0000063F0000063F00000C1F80000C0FC0001807E0003803F0007000FE01C0003FFF 800007FC001F1F7D9E26>II72 D76 DI<001FF80000FFFF0001F81F8007E007E00FC003F01F80 01F81F0000F83F0000FC7F0000FE7E00007E7E00007EFE00007FFE00007FFE00007FFE00 007FFE00007FFE00007FFE00007FFE00007FFE00007F7E00007E7F0000FE7F0000FE3F00 00FC3F8001FC1F8001F80FC003F007E007E001F81F8000FFFF00001FF800201F7D9E27> 79 D82 D86 D<07FC001FFF003F0F803F07C03F03E03F03E00C03E00003E0007FE007FBE01F 03E03C03E07C03E0F803E0F803E0F803E0FC05E07E0DE03FF8FE0FE07E17147F9319>97 DI<01FE0007 FF801F0FC03E0FC03E0FC07C0FC07C0300FC0000FC0000FC0000FC0000FC0000FC00007C 00007E00003E00603F00C01F81C007FF0001FC0013147E9317>I<0007F80007F80000F8 0000F80000F80000F80000F80000F80000F80000F80000F80000F801F8F80FFEF81F83F8 3E01F87E00F87C00F87C00F8FC00F8FC00F8FC00F8FC00F8FC00F8FC00F87C00F87C00F8 7E00F83E01F81F07F80FFEFF03F8FF18207E9F1D>I<01FE0007FF800F83C01E01E03E00 F07C00F07C00F8FC00F8FFFFF8FFFFF8FC0000FC0000FC00007C00007C00003E00181E00 180F807007FFE000FF8015147F9318>I<001F8000FFC001F3E003E7E003C7E007C7E007 C3C007C00007C00007C00007C00007C000FFFC00FFFC0007C00007C00007C00007C00007 C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C0003F FC003FFC0013207F9F10>I<01FC3C07FFFE0F079E1E03DE3E03E03E03E03E03E03E03E0 3E03E01E03C00F07800FFF0009FC001800001800001C00001FFF800FFFF007FFF81FFFFC 3C007C70003EF0001EF0001EF0001E78003C78003C3F01F80FFFE001FF00171E7F931A> II<1C003E00 3F007F003F003E001C00000000000000000000000000FF00FF001F001F001F001F001F00 1F001F001F001F001F001F001F001F001F001F001F00FFE0FFE00B217EA00E>I107 DI< FE0FE03F80FE1FF07FC01E70F9C3E01E407D01F01E807E01F01F807E01F01F007C01F01F 007C01F01F007C01F01F007C01F01F007C01F01F007C01F01F007C01F01F007C01F01F00 7C01F01F007C01F01F007C01F01F007C01F0FFE3FF8FFEFFE3FF8FFE27147D932C>II<01FF0007FFC0 1F83F03E00F83E00F87C007C7C007CFC007EFC007EFC007EFC007EFC007EFC007E7C007C 7C007C3E00F83E00F81F83F007FFC001FF0017147F931A>II114 D<0FE63FFE701E600EE006E006F800FFC07FF83FFC1FFE03FE001FC007 C007E007F006F81EFFFCC7F010147E9315>I<01800180018003800380038007800F803F 80FFFCFFFC0F800F800F800F800F800F800F800F800F800F800F860F860F860F860F8607 CC03F801F00F1D7F9C14>IIIII< FFE07F80FFE07F801F001C000F8018000F80180007C0300007C0300003E0600003E06000 01F0C00001F0C00001F9C00000F9800000FF8000007F0000007F0000003E0000003E0000 001C0000001C0000001800000018000078300000FC300000FC600000C0E00000E1C00000 7F8000001E000000191D7F931C>I<3FFFE03FFFE03C07C0380F80701F80603F00603E00 607C0000F80001F80003F00003E06007C0600F80601F80E03F00C03E01C07C03C0FFFFC0 FFFFC013147F9317>I E /Fo 27 122 df<70F8F8F87005057C840E>46 D<0001800000018000000180000003C0000003C0000003C0000005E0000005E000000DF0 000008F0000008F0000010F800001078000010780000203C0000203C0000203C0000401E 0000401E0000401E0000800F0000800F0000FFFF000100078001000780030007C0020003 C0020003C0040003E0040001E0040001E00C0000F00C0000F03E0001F8FF800FFF20237E A225>65 DI<0007E0100038183000E0063001C00170038000F0070000F00E0000701E000070 1C0000303C0000303C0000307C0000107800001078000010F8000000F8000000F8000000 F8000000F8000000F8000000F8000000F800000078000000780000107C0000103C000010 3C0000101C0000201E0000200E000040070000400380008001C0010000E0020000381C00 0007E0001C247DA223>I73 D<03FFF0001F00000F00000F00000F00000F00000F00 000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00 000F00000F00000F00000F00000F00000F00000F00700F00F80F00F80F00F80E00F01E00 401C0020380018700007C00014237EA119>I77 D<7FFFFFF8780780786007801840078008400780 0840078008C007800C800780048007800480078004800780040007800000078000000780 000007800000078000000780000007800000078000000780000007800000078000000780 000007800000078000000780000007800000078000000780000007800000078000000780 00000FC00003FFFF001E227EA123>84 DI<0FE0001838003C0C003C0E0018070000 070000070000070000FF0007C7001E07003C0700780700700700F00708F00708F00708F0 0F087817083C23900FC1E015157E9418>97 D<01FE000703000C07801C07803803007800 00700000F00000F00000F00000F00000F00000F00000F000007000007800403800401C00 800C010007060001F80012157E9416>99 D<0000E0000FE00001E00000E00000E00000E0 0000E00000E00000E00000E00000E00000E00000E00000E001F8E00704E00C02E01C01E0 3800E07800E07000E0F000E0F000E0F000E0F000E0F000E0F000E0F000E07000E07800E0 3800E01801E00C02E0070CF001F0FE17237EA21B>I<01FC000707000C03801C01C03801 C07801E07000E0F000E0FFFFE0F00000F00000F00000F00000F000007000007800203800 201C00400E008007030000FC0013157F9416>I<003C00C6018F038F030F070007000700 070007000700070007000700FFF807000700070007000700070007000700070007000700 070007000700070007000700070007807FF8102380A20F>I<00007001F198071E180E0E 181C07001C07003C07803C07803C07803C07801C07001C07000E0E000F1C0019F0001000 001000001800001800001FFE000FFFC00FFFE03800F0600030400018C00018C00018C000 186000306000303800E00E038003FE0015217F9518>I<0E0000FE00001E00000E00000E 00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E1F800E60C00E 80E00F00700F00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E 00700E00700E00700E00700E00700E0070FFE7FF18237FA21B>I<1C001E003E001E001C 00000000000000000000000000000000000E00FE001E000E000E000E000E000E000E000E 000E000E000E000E000E000E000E000E000E000E00FFC00A227FA10E>I<0E00FE001E00 0E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E000E000E000E000E000E00FFE00B237FA20E>108 D<0E1F80FE60C01E80E00F00700F00700E00700E00700E00700E00700E00700E00700E00 700E00700E00700E00700E00700E00700E00700E00700E0070FFE7FF18157F941B>110 D<01FC000707000C01801800C03800E0700070700070F00078F00078F00078F00078F000 78F00078F000787000707800F03800E01C01C00E038007070001FC0015157F9418>I<0E 1F00FE61C00E80600F00700E00380E003C0E001C0E001E0E001E0E001E0E001E0E001E0E 001E0E001E0E003C0E003C0E00380F00700E80E00E41C00E3F000E00000E00000E00000E 00000E00000E00000E00000E00000E0000FFE000171F7F941B>I<0E3CFE461E8F0F0F0F 060F000E000E000E000E000E000E000E000E000E000E000E000E000E000F00FFF010157F 9413>114 D<0F8830786018C018C008C008E008F0007F803FE00FF001F8003C801C800C 800CC00CC008E018D0308FC00E157E9413>I<02000200020002000600060006000E001E 003E00FFF80E000E000E000E000E000E000E000E000E000E000E000E040E040E040E040E 040E040708030801F00E1F7F9E13>I<0E0070FE07F01E00F00E00700E00700E00700E00 700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00F00E00F00601 7003827800FC7F18157F941B>II121 D E /Fp 31 120 df<0001FF0000001FFFC000007F80F00000FC00F80001F8 01F80003F803FC0007F003FC0007F003FC0007F003FC0007F001F80007F000F00007F000 000007F000000007F000000007F0000000FFFFFFFC00FFFFFFFC00FFFFFFFC0007F001FC 0007F001FC0007F001FC0007F001FC0007F001FC0007F001FC0007F001FC0007F001FC00 07F001FC0007F001FC0007F001FC0007F001FC0007F001FC0007F001FC0007F001FC0007 F001FC0007F001FC0007F001FC0007F001FC0007F001FC0007F001FC007FFF1FFFC07FFF 1FFFC07FFF1FFFC0222A7FA926>12 D<000E00001E00007E0007FE00FFFE00FFFE00F8FE 0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE 0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE 0000FE0000FE0000FE0000FE0000FE007FFFFE7FFFFE7FFFFE17277BA622>49 D<00FF800003FFF0000FFFFC001F03FE003800FF007C007F80FE003FC0FF003FC0FF003F E0FF001FE0FF001FE07E001FE03C003FE000003FE000003FC000003FC000007F8000007F 000000FE000000FC000001F8000003F0000003E00000078000000F0000001E0000003C00 E0007000E000E000E001C001C0038001C0070001C00FFFFFC01FFFFFC03FFFFFC07FFFFF C0FFFFFF80FFFFFF80FFFFFF801B277DA622>I<007F800003FFF00007FFFC000F81FE00 1F00FF003F80FF003F807F803F807F803F807F801F807F800F007F800000FF000000FF00 0000FE000001FC000001F8000007F00000FFC00000FFF0000001FC0000007E0000007F00 00007F8000003FC000003FC000003FE000003FE03C003FE07E003FE0FF003FE0FF003FE0 FF003FC0FF007FC07E007F807C007F003F01FE001FFFFC0007FFF00000FF80001B277DA6 22>I<00000E0000001E0000003E0000007E000000FE000000FE000001FE000003FE0000 077E00000E7E00000E7E00001C7E0000387E0000707E0000E07E0000E07E0001C07E0003 807E0007007E000E007E000E007E001C007E0038007E0070007E00E0007E00FFFFFFF8FF FFFFF8FFFFFFF80000FE000000FE000000FE000000FE000000FE000000FE000000FE0000 00FE00007FFFF8007FFFF8007FFFF81D277EA622>I68 D70 D<00007FE003000003FFFC0700001FFF FF0F00003FF00FFF0000FF8001FF0001FE0000FF0003F800003F0007F000003F000FF000 001F001FE000000F001FE000000F003FC000000F003FC0000007007FC0000007007F8000 0007007F8000000000FF8000000000FF8000000000FF8000000000FF8000000000FF8000 000000FF8000000000FF8000000000FF8000000000FF8001FFFFF87F8001FFFFF87F8001 FFFFF87FC00000FF003FC00000FF003FC00000FF001FE00000FF001FE00000FF000FF000 00FF0007F00000FF0003F80000FF0001FE0000FF0000FF8001FF00003FF007BF00001FFF FF1F000003FFFE0F0000007FF003002D297CA836>I73 D76 D78 D82 D<01FF800007FFF0000F81F8001FC07E001FC07E001FC03F000F80 3F8007003F8000003F8000003F8000003F80000FFF8000FFFF8007FC3F800FE03F803F80 3F803F003F807F003F80FE003F80FE003F80FE003F80FE003F807E007F807F00DF803F83 9FFC0FFF0FFC01FC03FC1E1B7E9A21>97 D<001FF80000FFFE0003F01F0007E03F800FC0 3F801F803F803F801F007F800E007F0000007F000000FF000000FF000000FF000000FF00 0000FF000000FF000000FF0000007F0000007F0000007F8000003F8001C01F8001C00FC0 038007E0070003F01E0000FFFC00001FE0001A1B7E9A1F>99 D<00003FF80000003FF800 00003FF800000003F800000003F800000003F800000003F800000003F800000003F80000 0003F800000003F800000003F800000003F800000003F800000003F800001FE3F80000FF FBF80003F03FF80007E00FF8000FC007F8001F8003F8003F8003F8007F0003F8007F0003 F8007F0003F800FF0003F800FF0003F800FF0003F800FF0003F800FF0003F800FF0003F8 00FF0003F8007F0003F8007F0003F8007F0003F8003F8003F8001F8003F8000F8007F800 07C00FF80003F03BFF8000FFF3FF80003FC3FF80212A7EA926>I<003FE00001FFF80003 F07E0007C01F000F801F801F800F803F800FC07F000FC07F0007C07F0007E0FF0007E0FF 0007E0FFFFFFE0FFFFFFE0FF000000FF000000FF0000007F0000007F0000007F0000003F 8000E01F8000E00FC001C007E0038003F81F0000FFFE00001FF0001B1B7E9A20>I<0007 F0003FFC00FE3E01F87F03F87F03F07F07F07F07F03E07F00007F00007F00007F00007F0 0007F00007F000FFFFC0FFFFC0FFFFC007F00007F00007F00007F00007F00007F00007F0 0007F00007F00007F00007F00007F00007F00007F00007F00007F00007F00007F00007F0 0007F00007F0007FFF807FFF807FFF80182A7EA915>I<00FF81F003FFE7F80FC1FE7C1F 80FC7C1F007C383F007E107F007F007F007F007F007F007F007F007F007F007F007F003F 007E001F007C001F80FC000FC1F8001FFFE00018FF800038000000380000003C0000003E 0000003FFFF8001FFFFF001FFFFF800FFFFFC007FFFFE01FFFFFF03E0007F07C0001F8F8 0000F8F80000F8F80000F8F80000F87C0001F03C0001E01F0007C00FC01F8003FFFE0000 7FF0001E287E9A22>II<07000FC01FE03FE03FE03FE01FE00FC0070000000000000000 00000000000000FFE0FFE0FFE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00F E00FE00FE00FE00FE00FE00FE00FE00FE00FE0FFFEFFFEFFFE0F2B7DAA14>I107 DII< FFC07F0000FFC1FFC000FFC787E0000FCE03F0000FD803F0000FD803F8000FF003F8000F F003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE0 03F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003 F8000FE003F8000FE003F800FFFE3FFF80FFFE3FFF80FFFE3FFF80211B7D9A26>I<003F E00001FFFC0003F07E000FC01F801F800FC03F800FE03F0007E07F0007F07F0007F07F00 07F0FF0007F8FF0007F8FF0007F8FF0007F8FF0007F8FF0007F8FF0007F8FF0007F87F00 07F07F0007F03F800FE03F800FE01F800FC00FC01F8007F07F0001FFFC00003FE0001D1B 7E9A22>II114 D<03FE300FFFF01E03F03800F0700070F00070F00070F80070FC0000 FFE0007FFE007FFF803FFFE01FFFF007FFF800FFF80003FC0000FC60007CE0003CF0003C F00038F80038FC0070FF01E0F7FFC0C1FF00161B7E9A1B>I<0070000070000070000070 0000F00000F00000F00001F00003F00003F00007F0001FFFF0FFFFF0FFFFF007F00007F0 0007F00007F00007F00007F00007F00007F00007F00007F00007F00007F00007F00007F0 3807F03807F03807F03807F03807F03803F03803F87001F86000FFC0001F8015267FA51B >II< FFFE7FFC0FFEFFFE7FFC0FFEFFFE7FFC0FFE0FE007E000E007F003F001C007F003F001C0 07F807F803C003F807F8038003F807F8038001FC0EFC070001FC0EFC070001FE1EFC0F00 00FE1C7E0E0000FE1C7E0E0000FF383F1E00007F383F1C00007F783F3C00003FF01FB800 003FF01FB800003FF01FF800001FE00FF000001FE00FF000000FC007E000000FC007E000 000FC007E00000078003C00000078003C0002F1B7F9A32>119 D E /Fq 59 124 df<00003FE00000E0100001803800038078000300780007003000070000 0007000000070000000E0000000E0000000E000000FFFFE0000E00E0001C01C0001C01C0 001C01C0001C01C0001C0380003803800038038000380380003807000038070000700700 0070071000700E2000700E2000700E2000E00E2000E0064000E0038000E0000000C00000 01C0000001C000003180000079800000F3000000620000003C0000001D29829F1A>12 D<1C3C3C3C3C040408081020204080060E7D840E>44 D<7FF0FFE07FE00C037D8A10>I< 70F8F8F0E005057B840E>I<000200020006000E003C00DC031C001C0038003800380038 007000700070007000E000E000E000E001C001C001C001C003800380038003800780FFF8 0F1E7B9D17>49 D<001F000061800080E00100E00200700220700420700410700820F008 20F00820F00840E00881E00703C0000380000700000C0000180000600000800003000004 00000800401000401000802001807E030047FF0041FE0080FC00807800141F7C9D17>I< 001F800060E00080700100300200380420380420380410380420700460700380600000E0 0001C000030000FE00001C00000600000700000780000780000780300780780780780780 F00F00800F00401E00401C0040380020E0001F8000151F7C9D17>I<0007C0001C200030 200060E000C1E00181E00380C00700000F00000E00001E00001E78001D84003E06003E07 003C07007C0780780780780780780780700F00700F00F00F00F00E00F01E00701C00601C 0070380030700010C0000F8000131F7B9D17>54 D<08E0100BF01017F8201FF8603E19C0 380E80200080600100400300800300000600000E00000C00001C00001C00003800003800 00700000700000F00000F00001E00001E00001E00003C00003C00003C00007C000078000 078000030000141F799D17>I<001F000061800080C00100600300600600600600600600 600E00C00F00800F818007C30007E40003F80001F80003FC00047E00183F00300F002007 00600700C00300C00300C00300800600800600C00C00C008004030003060001F8000131F 7B9D17>I<001F0000718000C0C00180C00380E00700E00F00E00F01E01E01E01E01E01E 01E01E01C01C03C01C03C01C03C01C07C01C0F800C0F8006378003C700000F00000E0000 0E00001C00601C00F03800F07000E0600080C0004380003E0000131F7B9D17>I<070F1F 1F0E0000000000000000000070F8F8F0E008147B930E>I<000002000000060000000600 00000E0000001E0000001E0000003F0000002F0000004F0000004F0000008F0000010F00 00010F0000020F0000020F0000040F00000C0F0000080F0000100F0000100F0000200F80 003FFF800040078000C00780008007800100078001000780020007800200078006000780 1E000F80FF807FF81D207E9F22>65 D<01FFFFC0001E00F0001E0078001E0038001E003C 003C003C003C003C003C003C003C003C0078007800780078007800F0007801E000F00780 00FFFE0000F00F8000F003C001E001C001E001E001E001E001E001E003C001E003C001E0 03C001E003C001C0078003C00780078007800F0007801E000F007800FFFFE0001E1F7D9E 20>I<0000FE0200078186001C004C0038003C0060003C00C0001C01C000180380001807 0000180F0000181E0000101E0000103C0000003C00000078000000780000007800000078 000000F0000000F0000000F0000000F0000000F000008070000080700000807000010038 00010038000200180004000C001800060020000381C00000FE00001F217A9F21>I<01FF FF80001E00E0001E0070001E0038001E001C003C001C003C000E003C000E003C000E0078 000E0078000E0078000E0078000E00F0001E00F0001E00F0001E00F0001E01E0003C01E0 003C01E0003C01E0007803C0007003C0007003C000E003C001C0078001C0078003800780 0E0007801C000F007000FFFFC0001F1F7D9E22>I<01FFFFFE001E001C001E000C001E00 04001E0004003C0004003C0004003C0004003C0004007808080078080000780800007818 0000F0300000FFF00000F0300000F0300001E0200001E0200001E0200001E0001003C000 2003C0002003C0004003C00040078000800780018007800100078007000F001F00FFFFFE 001F1F7D9E1F>I<01FFFFFC001E0038001E0018001E0008001E0008003C0008003C0008 003C0008003C00080078001000780800007808000078080000F0100000F0300000FFF000 00F0300001E0200001E0200001E0200001E0200003C0000003C0000003C0000003C00000 078000000780000007800000078000000F800000FFF800001E1F7D9E1E>I<0000FC0400 07030C001C00980030007800E0007801C000380380003003800030070000300E0000301E 0000201E0000203C0000003C00000078000000780000007800000078000000F0000000F0 00FFF0F0000780F0000780F0000F0070000F0070000F0070000F0070001E0038001E0018 003E001C002E000E00CC000383040000FC00001E217A9F23>I<01FFF3FFE0001F003E00 001E003C00001E003C00001E003C00003C007800003C007800003C007800003C00780000 7800F000007800F000007800F000007800F00000F001E00000FFFFE00000F001E00000F0 01E00001E003C00001E003C00001E003C00001E003C00003C007800003C007800003C007 800003C007800007800F000007800F000007800F000007800F00000F801F0000FFF1FFE0 00231F7D9E22>I<01FFF0001F00001E00001E00001E00003C00003C00003C00003C0000 780000780000780000780000F00000F00000F00000F00001E00001E00001E00001E00003 C00003C00003C00003C0000780000780000780000780000F8000FFF800141F7D9E12>I< 001FFF0000F80000F00000F00000F00001E00001E00001E00001E00003C00003C00003C0 0003C0000780000780000780000780000F00000F00000F00000F00001E00001E00301E00 781E00F83C00F83C00F0780080700040E00021C0001F000018207D9E18>I<01FFF80000 1F0000001E0000001E0000001E0000003C0000003C0000003C0000003C00000078000000 780000007800000078000000F0000000F0000000F0000000F0000001E0000001E0000001 E0000001E0008003C0010003C0010003C0030003C00200078006000780060007800C0007 801C000F007800FFFFF800191F7D9E1D>76 D<01FE00007FC0001E0000FC00001E0000F8 0000170001780000170001780000270002F00000270004F00000270004F00000270008F0 0000470009E00000470011E00000470021E00000470021E00000870043C00000838043C0 0000838083C00000838083C0000103810780000103820780000103820780000103840780 000203840F00000203880F00000203900F00000203900F00000401E01E00000401E01E00 000401C01E00000C01801E00001C01803E0000FF8103FFC0002A1F7D9E29>I<01FF007F E0001F000F00001F0004000017800400001780040000278008000023C008000023C00800 0023C008000041E010000041E010000041F010000040F010000080F02000008078200000 80782000008078200001003C400001003C400001003C400001001E400002001E80000200 1E800002000F800002000F800004000F0000040007000004000700000C000700001C0002 0000FF80020000231F7D9E22>I<0001FC0000070700001C01C0003000E000E0006001C0 00700380007007800038070000380E0000381E0000381C0000383C0000383C0000387800 0078780000787800007878000078F00000F0F00000F0F00000E0F00001E0F00001C0F000 03C0700003807000070078000F0038001E0038003C001C0070000E00E0000783800001FC 00001D217A9F23>I<01FFFF80001E00E0001E0070001E0038001E003C003C003C003C00 3C003C003C003C003C0078007800780078007800F0007800E000F003C000F00F0000FFFC 0000F0000001E0000001E0000001E0000001E0000003C0000003C0000003C0000003C000 00078000000780000007800000078000000F800000FFF000001E1F7D9E1F>I<01FFFF00 001E03C0001E00E0001E0070001E0078003C0078003C0078003C0078003C0078007800F0 007800F0007801E0007801C000F0070000F01E0000FFF00000F0380001E01C0001E01E00 01E00E0001E00F0003C01E0003C01E0003C01E0003C01E0007803C0007803C0807803C08 07803C100F801C10FFF00C20000007C01D207D9E21>82 D<0007E040001C18C000300580 0060038000C0038001C00180018001000380010003800100038001000380000003C00000 03C0000003F8000001FF800001FFE000007FF000001FF0000001F8000000780000007800 000038000000380020003800200038002000300060007000600060006000E0007000C000 E8038000C606000081F800001A217D9F1A>I<0FFFFFF01E0780E0180780201007802020 078020200F0020600F0020400F0020400F0020801E0040001E0000001E0000001E000000 3C0000003C0000003C0000003C00000078000000780000007800000078000000F0000000 F0000000F0000000F0000001E0000001E0000001E0000001E0000003E00000FFFF00001C 1F789E21>I<7FFC1FF807C003C00780010007800100078001000F0002000F0002000F00 02000F0002001E0004001E0004001E0004001E0004003C0008003C0008003C0008003C00 080078001000780010007800100078001000F0002000F0002000F0002000F0004000F000 4000700080007001000030020000380400000C18000007E000001D20779E22>II<00F1800389C00707800E0380 1C03803C0380380700780700780700780700F00E00F00E00F00E00F00E20F01C40F01C40 703C40705C40308C800F070013147C9317>97 D<07803F8007000700070007000E000E00 0E000E001C001C001CF01D0C3A0E3C0E380F380F700F700F700F700FE01EE01EE01EE01C E03CE038607060E031C01F0010207B9F15>I<007E0001C1000300800E07801E07801C07 003C0200780000780000780000F00000F00000F00000F00000F000007001007002003004 0018380007C00011147C9315>I<0000780003F80000700000700000700000700000E000 00E00000E00000E00001C00001C000F1C00389C00707800E03801C03803C038038070078 0700780700780700F00E00F00E00F00E00F00E20F01C40F01C40703C40705C40308C800F 070015207C9F17>I<007C01C207010E011C013C013802780C7BF07C00F000F000F000F0 007000700170023804183807C010147C9315>I<00007800019C00033C00033C00071800 0700000700000E00000E00000E00000E00000E0001FFE0001C00001C00001C00001C0000 380000380000380000380000380000700000700000700000700000700000700000E00000 E00000E00000E00000C00001C00001C0000180003180007B0000F300006600003C000016 29829F0E>I<003C6000E27001C1E00380E00700E00F00E00E01C01E01C01E01C01E01C0 3C03803C03803C03803C03803C07003C07001C0F001C17000C2E0003CE00000E00000E00 001C00001C00301C00783800F0700060E0003F8000141D7E9315>I<01E0000FE00001C0 0001C00001C00001C000038000038000038000038000070000070000071E000763000E81 800F01C00E01C00E01C01C03801C03801C03801C0380380700380700380700380E10700E 20700C20701C20700C40E00CC060070014207D9F17>I<00C001E001E001C00000000000 0000000000000000000E003300230043804300470087000E000E000E001C001C001C0038 40388030807080310033001C000B1F7C9E0E>I<0001800003C00003C000038000000000 0000000000000000000000000000000000003C0000460000870000870001070001070002 0E00000E00000E00000E00001C00001C00001C00001C0000380000380000380000380000 700000700000700000700000E00000E00030E00079C000F180006300003C00001228829E 0E>I<01E0000FE00001C00001C00001C00001C000038000038000038000038000070000 0700000703C00704200E08E00E11E00E21E00E40C01C80001D00001E00001FC00038E000 387000387000383840707080707080707080703100E03100601E0013207D9F15>I<03C0 1FC0038003800380038007000700070007000E000E000E000E001C001C001C001C003800 3800380038007000700070007100E200E200E200E200640038000A207C9F0C>I<1C0F80 F0002630C318004740640C004780680E004700700E004700700E008E00E01C000E00E01C 000E00E01C000E00E01C001C01C038001C01C038001C01C038001C01C070803803807100 3803806100380380E10038038062007007006600300300380021147C9325>I<1C0F8026 30C04740604780604700704700708E00E00E00E00E00E00E00E01C01C01C01C01C01C01C 03843803883803083807083803107003303001C016147C931A>I<007C0001C300030180 0E01C01E01C01C01E03C01E07801E07801E07801E0F003C0F003C0F003C0F00780F00700 700F00700E0030180018700007C00013147C9317>I<01C1E002621804741C04781C0470 1E04701E08E01E00E01E00E01E00E01E01C03C01C03C01C03C01C0380380780380700380 E003C1C0072380071E000700000700000E00000E00000E00000E00001C00001C0000FFC0 00171D809317>I<00F0400388C00705800E03801C03803C038038070078070078070078 0700F00E00F00E00F00E00F00E00F01C00F01C00703C00705C0030B8000F380000380000 380000700000700000700000700000E00000E0000FFE00121D7C9315>I<1C1E00266100 4783804787804707804703008E00000E00000E00000E00001C00001C00001C00001C0000 38000038000038000038000070000030000011147C9313>I<00FC030206010C030C070C 060C000F800FF007F803FC003E000E700EF00CF00CE008401020601F8010147D9313>I< 018001C0038003800380038007000700FFF007000E000E000E000E001C001C001C001C00 3800380038003820704070407080708031001E000C1C7C9B0F>I<0E00C03300E02301C0 4381C04301C04701C08703800E03800E03800E03801C07001C07001C07001C07101C0E20 180E20180E201C1E200C264007C38014147C9318>I<0E03803307802307C04383C04301 C04700C08700800E00800E00800E00801C01001C01001C01001C02001C02001C04001C04 001C08000E300003C00012147C9315>I<0E00C1C03300E3C02301C3E04381C1E04301C0 E04701C060870380400E0380400E0380400E0380401C0700801C0700801C0700801C0701 001C0701001C0602001C0F02000C0F04000E13080003E1F0001B147C931E>I<0383800C C4401068E01071E02071E02070C040E00000E00000E00000E00001C00001C00001C00001 C040638080F38080F38100E5810084C60078780013147D9315>I<0E00C03300E02301C0 4381C04301C04701C08703800E03800E03800E03801C07001C07001C07001C07001C0E00 180E00180E001C1E000C3C0007DC00001C00001C00003800F03800F07000E06000C0C000 4380003E0000131D7C9316>I<01C04003E08007F1800C1F000802000004000008000010 000020000040000080000100000200000401000802001002003E0C0063FC0041F80080E0 0012147D9313>II E /Fr 88 125 df<00008000000001C0000000 01C000000003E000000003E000000005F000000004F000000008F8000000087800000010 7C000000103C000000203E000000201E000000401F000000400F000000800F8000008007 8000010007C000010003C000020003E000020001E000040001F000040000F000080000F8 0008000078001000007C001000003C002000003E002000001E007FFFFFFF007FFFFFFF00 FFFFFFFF8021207E9F26>1 D<003FC00000E0700003801C0007000E000F000F001E0007 803E0007C03C0003C07C0003E07C0003E07C0003E07C0003E07C0003E07C0003E07C0003 E03C0003C03E0007C01E0007801E0007800E0007000F000F0007000E0003000C0003801C 0001801800818018108080101040C03020404020207FC03FE03FC03FC03FC03FC01C207E 9F21>10 D<001F83E000F06E3001C078780380F8780300F0300700700007007000070070 0007007000070070000700700007007000FFFFFF80070070000700700007007000070070 000700700007007000070070000700700007007000070070000700700007007000070070 0007007000070070000700700007007000070070007FE3FF001D20809F1B>I<003F0000 E0C001C0C00381E00701E00701E0070000070000070000070000070000070000FFFFE007 00E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E007 00E00700E00700E00700E00700E00700E07FC3FE1720809F19>I<003FE000E0E001C1E0 0381E00700E00700E00700E00700E00700E00700E00700E00700E0FFFFE00700E00700E0 0700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E0 0700E00700E00700E00700E07FE7FE1720809F19>I<001F81F80000F04F040001C07C06 000380F80F000300F00F000700F00F000700700000070070000007007000000700700000 07007000000700700000FFFFFFFF00070070070007007007000700700700070070070007 007007000700700700070070070007007007000700700700070070070007007007000700 7007000700700700070070070007007007000700700700070070070007007007007FE3FE 3FF02420809F26>I<7038F87CFC7EFC7E743A0402040204020804080410081008201040 200F0E7E9F17>34 D<007800000084000001840000030200000702000007020000070200 0007020000070400000704000007080000070800000310000003A00FFC03C003E0038001 C001C0008001C0010003E0010004E0020008F00200187004003078080070380800701C10 00F01E1000F00E2000F0074000F003C0087003C0087801C010380670301C18386007E00F 801E227EA023>38 D<70F8FCFC74040404080810102040060E7C9F0D>I<002000400080 0100020006000C000C00180018003000300030007000600060006000E000E000E000E000 E000E000E000E000E000E000E000E0006000600060007000300030003000180018000C00 0C000600020001000080004000200B2E7DA112>I<800040002000100008000C00060006 000300030001800180018001C000C000C000C000E000E000E000E000E000E000E000E000 E000E000E000E000C000C000C001C001800180018003000300060006000C000800100020 00400080000B2E7DA112>I<000600000006000000060000000600000006000000060000 000600000006000000060000000600000006000000060000000600000006000000060000 FFFFFFF0FFFFFFF000060000000600000006000000060000000600000006000000060000 00060000000600000006000000060000000600000006000000060000000600001C207D9A 23>43 D<70F8FCFC74040404080810102040060E7C840D>II<70 F8F8F87005057C840D>I<000100030003000600060006000C000C000C00180018001800 300030003000600060006000C000C000C00180018001800300030003000600060006000C 000C000C00180018001800300030003000600060006000C000C000C000102D7DA117>I< 03F0000E1C001C0E00180600380700700380700380700380700380F003C0F003C0F003C0 F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0700380700380 7003807807803807001806001C0E000E1C0003F000121F7E9D17>I<018003800F80F380 038003800380038003800380038003800380038003800380038003800380038003800380 03800380038003800380038007C0FFFE0F1E7C9D17>I<03F0000C1C00100E0020070040 0780800780F007C0F803C0F803C0F803C02007C00007C0000780000780000F00000E0000 1C0000380000700000600000C0000180000300000600400C00401800401000803FFF807F FF80FFFF80121E7E9D17>I<03F0000C1C00100E00200F00780F80780780780780380F80 000F80000F00000F00000E00001C0000380003F000003C00000E00000F00000780000780 0007C02007C0F807C0F807C0F807C0F00780400780400F00200E001C3C0003F000121F7E 9D17>I<000600000600000E00000E00001E00002E00002E00004E00008E00008E00010E 00020E00020E00040E00080E00080E00100E00200E00200E00400E00C00E00FFFFF0000E 00000E00000E00000E00000E00000E00000E0000FFE0141E7F9D17>I<1803001FFE001F FC001FF8001FE00010000010000010000010000010000010000011F000161C00180E0010 07001007800003800003800003C00003C00003C07003C0F003C0F003C0E0038040038040 0700200600100E000C380003E000121F7E9D17>I<007C000182000701000E03800C0780 1C0780380300380000780000700000700000F1F000F21C00F40600F80700F80380F80380 F003C0F003C0F003C0F003C0F003C07003C07003C07003803803803807001807000C0E00 061C0001F000121F7E9D17>I<4000007FFFC07FFF807FFF804001008002008002008004 0000080000080000100000200000200000400000400000C00000C00001C0000180000380 00038000038000038000078000078000078000078000078000078000078000030000121F 7D9D17>I<03F0000C0C001006003003002001806001806001806001807001807803003E 03003F06001FC8000FF00003F80007FC000C7E00103F00300F806003804001C0C001C0C0 00C0C000C0C000C0C000806001802001001002000C0C0003F000121F7E9D17>I<03F000 0E18001C0C00380600380700700700700380F00380F00380F003C0F003C0F003C0F003C0 F003C07007C07007C03807C0180BC00E13C003E3C0000380000380000380000700300700 780600780E00700C002018001070000FC000121F7E9D17>I<70F8F8F870000000000000 0000000070F8F8F87005147C930D>I<70F8F8F8700000000000000000000070F0F8F878 080808101010202040051D7C930D>I<7FFFFFE0FFFFFFF0000000000000000000000000 0000000000000000000000000000000000000000FFFFFFF07FFFFFE01C0C7D9023>61 D<0FC0307040384038E03CF03CF03C603C0038007000E000C00180018001000300020002 0002000200020002000000000000000000000007000F800F800F8007000E207D9F15>63 D<000100000003800000038000000380000007C0000007C0000007C0000009E0000009E0 000009E0000010F0000010F0000010F00000207800002078000020780000403C0000403C 0000403C0000801E0000801E0000FFFE0001000F0001000F0001000F0002000780020007 8002000780040003C00E0003C01F0007E0FFC03FFE1F207F9F22>65 DI<000FC040007030C001C009C0 038005C0070003C00E0001C01E0000C01C0000C03C0000C07C0000407C00004078000040 F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000 780000007C0000407C0000403C0000401C0000401E0000800E0000800700010003800200 01C0040000703800000FC0001A217D9F21>IIII<000FE0200078186000E004E0038002E0070001E0 0F0000E01E0000601E0000603C0000603C0000207C00002078000020F8000000F8000000 F8000000F8000000F8000000F8000000F8000000F8007FFCF80003E0780001E07C0001E0 3C0001E03C0001E01E0001E01E0001E00F0001E0070001E0038002E000E0046000781820 000FE0001E217D9F24>III<0FFFC0007C 00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C 00003C00003C00003C00003C00003C00003C00003C00003C00003C00203C00F83C00F83C 00F83C00F0380040780040700030E0000F800012207E9E17>IIIII<001F800000F0F00001C0380007801E000F000F000E0007001E00 07803C0003C03C0003C07C0003E0780001E0780001E0F80001F0F80001F0F80001F0F800 01F0F80001F0F80001F0F80001F0F80001F0F80001F0780001E07C0003E07C0003E03C00 03C03C0003C01E0007800E0007000F000F0007801E0001C0380000F0F000001F80001C21 7D9F23>II<001F800000F0F000 01C0380007801E000F000F000E0007001E0007803C0003C03C0003C07C0003E07C0003E0 780001E0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0 F80001F0780001E0780001E07C0003E03C0003C03C0F03C01E1087800E20C7000F20CF00 07A0FE0001E0F80000F0F010001FF01000007010000078300000387000003FF000003FE0 00001FE000000FC0000007801C297D9F23>II<07E0800C1980100780300380600180600180E00180E00080E000 80E00080F00000F000007800007F00003FF0001FFC000FFE0003FF00001F800007800003 C00003C00001C08001C08001C08001C08001C0C00180C00380E00300F00600CE0C0081F8 0012217D9F19>I<7FFFFFE0780F01E0600F0060400F0020400F0020C00F0030800F0010 800F0010800F0010800F0010000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000001F800007FFFE001C1F7E9E21>IIII<7FF83FF8 0FE00FC007C0070003C0020001E0040001F00C0000F0080000781000007C1000003C2000 003E4000001E4000000F8000000F8000000780000003C0000007E0000005E0000009F000 0018F8000010780000207C0000603C0000401E0000801F0001800F0001000780020007C0 070003C01F8007E0FFE01FFE1F1F7F9E22>II91 D<08041008201020104020402080 4080408040B85CFC7EFC7E7C3E381C0F0E7B9F17>II<0C001E0033006180C0C080400A067A9E17>I<081020204040808080B8FCFC7C38060E 7D9F0D>96 D<1FE000303000781800781C00300E00000E00000E00000E0000FE00078E00 1E0E00380E00780E00F00E10F00E10F00E10F01E10781E103867200F83C014147E9317> I<0E0000FE00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00 000E3E000EC3800F01C00F00E00E00E00E00700E00700E00780E00780E00780E00780E00 780E00780E00700E00700E00E00F00E00D01C00CC300083E0015207F9F19>I<03F80E0C 1C1E381E380C70007000F000F000F000F000F000F00070007000380138011C020E0C03F0 10147E9314>I<000380003F800003800003800003800003800003800003800003800003 8000038000038003E380061B801C0780380380380380700380700380F00380F00380F003 80F00380F00380F003807003807003803803803807801C07800E1B8003E3F815207E9F19 >I<03F0000E1C001C0E00380700380700700700700380F00380F00380FFFF80F00000F0 0000F000007000007000003800801800800C010007060001F80011147F9314>I<007C00 C6018F038F07060700070007000700070007000700FFF007000700070007000700070007 00070007000700070007000700070007000700070007007FF01020809F0E>I<0000E003 E3300E3C301C1C30380E00780F00780F00780F00780F00780F00380E001C1C001E380033 E0002000002000003000003000003FFE001FFF800FFFC03001E0600070C00030C00030C0 0030C000306000603000C01C038003FC00141F7F9417>I<0E0000FE00000E00000E0000 0E00000E00000E00000E00000E00000E00000E00000E00000E3E000E43000E81800F01C0 0F01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C0FFE7FC16207F9F19>I<1C003E003E003E001C0000000000000000 00000000000E007E000E000E000E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E00FFC00A1F809E0C>I<00E001F001F001F000E000000000000000000000 0000007007F000F000700070007000700070007000700070007000700070007000700070 00700070007000700070007000706070F060F0C061803F000C28829E0E>I<0E0000FE00 000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E0FF00E03 C00E03000E02000E04000E08000E10000E30000E70000EF8000F38000E1C000E1E000E0E 000E07000E07800E03800E03C00E03E0FFCFF815207F9F18>I<0E00FE000E000E000E00 0E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E00FFE00B20809F0C>I<0E1F01F000FE618618000E 81C81C000F00F00E000F00F00E000E00E00E000E00E00E000E00E00E000E00E00E000E00 E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E0 0E000E00E00E000E00E00E00FFE7FE7FE023147F9326>I<0E3E00FE43000E81800F01C0 0F01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C0FFE7FC16147F9319>I<01F800070E001C03803801C03801C07000 E07000E0F000F0F000F0F000F0F000F0F000F0F000F07000E07000E03801C03801C01C03 80070E0001F80014147F9317>I<0E3E00FEC3800F01C00F00E00E00E00E00F00E00700E 00780E00780E00780E00780E00780E00780E00700E00F00E00E00F01E00F01C00EC3000E 3E000E00000E00000E00000E00000E00000E00000E00000E0000FFE000151D7F9319>I< 03E0800619801C05803C0780380380780380700380F00380F00380F00380F00380F00380 F003807003807803803803803807801C0B800E138003E380000380000380000380000380 000380000380000380000380003FF8151D7E9318>I<0E78FE8C0F1E0F1E0F0C0E000E00 0E000E000E000E000E000E000E000E000E000E000E000E00FFE00F147F9312>I<1F9030 704030C010C010C010E00078007F803FE00FF00070803880188018C018C018E030D0608F 800D147E9312>I<020002000200060006000E000E003E00FFF80E000E000E000E000E00 0E000E000E000E000E000E000E080E080E080E080E080610031001E00D1C7F9B12>I<0E 01C0FE1FC00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C00E01C00E03C00603C0030DC001F1FC16147F9319>III<7FC3FC0F01E00701C007018003810001C20000E40000EC00007800003800003C 00007C00004E000087000107000303800201C00601E01E01E0FF07FE1714809318>II<3FFF380E200E201C40384078407000 E001E001C00380078007010E011E011C0338027006700EFFFE10147F9314>III E end %%EndProlog %%BeginSetup %%Feature: *Resolution 300dpi TeXDict begin %%EndSetup %%Page: 29 1 29 0 bop 0 192 a Fn(V)o(C)16 b(dimension)23 b Fr(A)18 b(measure)g(of)f(the)h(complexit)o(y)g(of)g(a)f(mo)q(del.)28 b(Kno)o(wledge)19 b(of)e(the)h(V)o(C)f(dimension)i(p)q(er-)114 248 y(mits)e(an)f(estimate)h(to)f(b)q(e)i(made)f(of)f(the)h (di\013erence)h(b)q(et)o(w)o(een)f(p)q(erformance)g(on)g(the)g (training)g(set)g(and)114 304 y(p)q(erformance)e(on)g(a)g(test)g(set.) 952 2775 y(29)p eop %%Page: 28 2 28 1 bop 0 192 a Fp(De\014ning)22 b(terms)0 293 y Fn(Classi\014cation)j Fr(A)14 b(learning)h(problem)f(in)h(whic)o(h)g(the)f(goal)f(is)h(to)g (assign)g(input)g(v)o(ectors)g(to)f(one)h(of)f(a)h(n)o(um)o(b)q(er)114 349 y(of)g(\(usually)j(m)o(utually)f(exclusiv)o(e\))g(classes.)0 443 y Fn(Boltzmann)j(mac)o(hine)k Fr(An)17 b(undirected)i(net)o(w)o (ork)c(of)i(discrete)h(v)m(alued)g(random)e(v)m(ariables,)j(where)e(an) f(en-)114 500 y(ergy)f(function)h(is)g(asso)q(ciated)g(with)f(eac)o(h)h (of)f(the)g(links,)i(and)e(for)g(whic)o(h)i(a)e(probabilit)o(y)h (distribution)h(is)114 556 y(de\014ned)f(b)o(y)f(the)h(Boltzmann)f (distribution.)0 650 y Fn(Cost)i(function)24 b Fr(A)12 b(function)h(of)f(the)g(adaptiv)o(e)h(parameters)e(of)h(a)g(mo)q(del)h (whose)f(minim)o(um)h(is)g(used)f(to)g(de\014ne)114 706 y(suitable)18 b(v)m(alues)g(for)f(those)g(parameters.)25 b(It)17 b(ma)o(y)g(consist)g(of)g(a)g(lik)o(eliho)q(o)q(d)j(function)e (and)g(additional)114 763 y(terms.)0 857 y Fn(Decision)h(tree)j Fr(A)16 b(net)o(w)o(ork)f(that)h(p)q(erforms)f(a)h(sequence)h(of)f (classi\014catory)h(decisions)g(on)f(an)h(input)g(v)o(ector)114 913 y(and)e(pro)q(duces)h(an)f(output)g(v)o(ector)g(that)f(is)i (conditional)h(on)e(the)g(outcome)g(of)g(the)g(decision)i(sequence.)0 1007 y Fn(Densit)o(y)g(estimation)24 b Fr(The)17 b(problem)g(of)g(mo)q (deling)h(a)e(probabilit)o(y)i(distribution)g(from)e(a)g(\014nite)i (set)e(of)h(ex-)114 1063 y(amples)e(dra)o(wn)g(from)g(that)f (distribution.)0 1157 y Fn(Discriminan)o(t)k(function)24 b Fr(A)d(function)g(of)e(the)h(input)i(v)o(ector)d(whic)o(h)i(can)f(b)q (e)h(used)g(to)e(assign)i(inputs)g(to)114 1214 y(classes)15 b(in)h(a)f(classi\014cation)i(problem.)0 1308 y Fn(Hidden)h(Mark)o(o)o (v)d(mo)q(del)24 b Fr(A)14 b(graphical)h(probabilistic)h(mo)q(del)f(c)o (haracterized)f(b)o(y)g(a)g(state)f(v)o(ector,)g(an)h(out-)114 1364 y(put)h(v)o(ector,)f(a)h(state)f(transition)i(matrix,)e(an)h (emission)i(matrix)e(and)g(an)g(initial)i(state)e(distribution.)0 1458 y Fn(Lik)o(eliho)q(o)q(d)20 b(function)k Fr(The)15 b(probabilit)o(y)h(of)e(observing)i(a)e(particular)i(data)e(set)h (under)g(the)g(assumption)g(of)114 1514 y(a)d(giv)o(en)i(parametrized)f (mo)q(del,)h(expressed)f(as)g(a)g(function)g(of)g(the)g(adaptiv)o(e)g (parameters)f(of)g(the)h(mo)q(del.)0 1608 y Fn(Mixture)k(mo)q(del)23 b Fr(A)13 b(probabilit)o(y)g(mo)q(del)g(whic)o(h)g(consists)g(of)e(a)h (linear)i(com)o(bination)f(of)e(simpler)j(comp)q(onen)o(t)114 1665 y(probabilit)o(y)i(mo)q(dels.)0 1758 y Fn(Multila)o(y)o(er)i(p)q (erceptron)23 b Fr(The)11 b(most)f(common)h(form)f(of)h(neural)h(net)o (w)o(ork)e(mo)q(del,)i(consisting)g(of)f(successiv)o(e)114 1815 y(linear)16 b(transformations)e(follo)o(w)o(ed)h(b)o(y)g(pro)q (cessing)i(with)e(nonlinear)i(activ)m(ation)f(functions.)0 1909 y Fn(Ov)o(er\014tting)23 b Fr(The)16 b(problem)g(in)g(whic)o(h)h (a)e(mo)q(del)i(whic)o(h)f(is)g(to)q(o)f(complex)h(captures)g(to)q(o)f (m)o(uc)o(h)g(of)h(the)f(noise)114 1965 y(in)h(the)f(data,)f(leading)j (to)d(p)q(o)q(or)h(generalization.)0 2059 y Fn(Radial)20 b(basis)d(function)i(net)o(w)o(ork)i Fr(A)14 b(common)f(net)o(w)o(ork)f (mo)q(del)j(consisting)f(of)f(a)g(linear)i(com)o(bination)f(of)114 2115 y(basis)j(functions)h(eac)o(h)f(of)f(whic)o(h)i(is)f(a)g(function) g(of)g(the)g(di\013erence)h(b)q(et)o(w)o(een)f(the)g(input)h(v)o(ector) e(and)h(a)114 2172 y(cen)o(ter)e(v)o(ector.)0 2266 y Fn(Regression)22 b Fr(A)d(learning)h(problem)f(in)h(whic)o(h)f(the)g (goal)g(is)g(to)f(map)g(eac)o(h)h(input)h(v)o(ector)e(to)g(a)g(real-v)m (alued)114 2322 y(output)d(v)o(ector.)0 2416 y Fn(Regularization)26 b Fr(A)19 b(tec)o(hnique)i(for)d(con)o(trolling)j(mo)q(del)f(complexit) o(y)g(and)g(impro)o(ving)f(generalization)i(b)o(y)114 2472 y(the)15 b(addition)h(of)f(a)g(p)q(enalt)o(y)h(term)e(to)h(the)g (cost)g(function.)952 2775 y(28)p eop %%Page: 27 3 27 2 bop 0 192 a Fr(Shac)o(h)o(ter,)12 b(R.,)g(Andersen,)h(S.,)f(and)g (Szolo)o(vits,)h(P)l(.)e(1994.)18 b(Global)13 b(conditioning)g(for)f (probabilistic)i(inference)f(in)91 248 y(b)q(elief)i(net)o(w)o(orks.)j (In)c Fq(Unc)n(ertainty)f(in)h(A)o(rti\014cial)g(Intel)r(ligenc)n(e:)j (Pr)n(o)n(c)n(e)n(e)n(dings)12 b(of)j(the)f(T)m(enth)f(Confer)n(enc)n (e)p Fr(,)91 304 y(p.)20 b(514{522.)e(Seattle,)d(W)-5 b(A.)0 398 y(Shac)o(h)o(ter,)15 b(R.)h(and)g(Kenley)l(,)h(C.)e(1989.)20 b(Gaussian)c(in\015uence)h(diagrams.)k Fq(Management)16 b(Scienc)n(e)p Fr(,)e(35\(5\):527{)91 455 y(550.)0 549 y(Spiegelhalter,)25 b(D.,)d(Da)o(wid,)g(A.,)h(Lauritzen,)h(S.,)e(and)g (Co)o(w)o(ell,)h(R.)f(1993.)37 b(Ba)o(y)o(esian)22 b(analysis)g(in)h (exp)q(ert)91 605 y(systems.)c Fq(Statistic)n(al)d(Scienc)n(e)p Fr(,)d(8\(3\):219{28)o(3.)0 699 y(T)l(arassenk)o(o,)h(L.)i(1995.)k(No)o (v)o(elt)o(y)c(detection)g(for)f(the)h(iden)o(ti\014cation)i(of)d (masses)g(in)i(mammograms.)i Fq(Pr)n(o)n(c)n(e)n(e)n(d-)91 755 y(ings)c(F)m(ourth)i(IEE)e(International)g(Confer)n(enc)n(e)f(on)i (A)o(rti\014cial)g(Neur)n(al)g(Networks)p Fr(,)f(4:442{447.)0 849 y(V)l(apnik,)h(V.)f(N.)g(1995.)j Fq(The)e(Natur)n(e)h(of)f (Statistic)n(al)g(L)n(e)n(arning)e(The)n(ory)p Fr(.)20 b(Springer-V)l(erlag,)c(New)f(Y)l(ork.)0 992 y Fp(F)-6 b(urther)24 b(information)0 1094 y Fr(In)f(this)g(c)o(hapter)f(w)o(e)g (ha)o(v)o(e)g(emphasized)i(the)e(links)i(b)q(et)o(w)o(een)f(neural)g (net)o(w)o(orks)e(and)i(statistical)f(pattern)0 1150 y(recognition.)e(A)13 b(more)g(extensiv)o(e)g(treatmen)o(t)f(from)h (the)g(same)f(p)q(ersp)q(ectiv)o(e)j(can)e(b)q(e)h(found)f(in)h ([Bishop,)g(1995].)0 1207 y(F)l(or)g(a)h(view)g(of)g(recen)o(t)g (researc)o(h)f(in)i(the)f(\014eld,)h(the)f(pro)q(ceedings)h(of)f(the)g (ann)o(ual)g(NIPS)h(\(Neural)f(Information)0 1263 y(Pro)q(cessing)h (Systems;)e(MIT)h(Press\))g(conferences)h(are)f(highly)i(recommended.) 91 1320 y(Neural)h(computing)g(is)g(no)o(w)f(a)g(v)o(ery)h(broad)f (\014eld)i(and)e(there)h(are)f(man)o(y)g(topics)h(whic)o(h)h(ha)o(v)o (e)e(not)g(b)q(een)0 1376 y(discussed)e(for)f(lac)o(k)g(of)f(space.)20 b(Here)14 b(w)o(e)g(aim)g(to)f(pro)o(vide)i(a)e(brief)i(o)o(v)o(erview) f(of)f(some)h(of)f(the)h(more)g(signi\014can)o(t)0 1432 y(omissions,)h(and)h(to)e(giv)o(e)i(p)q(oin)o(ters)f(to)g(the)g (literature.)91 1489 y(The)g(resurgence)h(of)e(in)o(terest)i(in)g (neural)f(net)o(w)o(orks)f(during)i(the)f(1980's)f(w)o(as)g(due)i(in)g (large)f(part)f(to)h(w)o(ork)0 1545 y(on)g(the)g(statistical)h(mec)o (hanics)g(of)f(fully)h(connected)h(net)o(w)o(orks)d(ha)o(ving)h (symmetric)g(connections)i(\(i.e.)e(if)g(unit)0 1602 y Fm(i)h Fr(sends)h(a)f(connection)i(to)e(unit)h Fm(j)i Fr(then)e(there)f(is)h(also)g(a)f(connection)i(from)d(unit)j Fm(j)g Fr(bac)o(k)f(to)f(unit)h Fm(i)f Fr(with)h(the)0 1658 y(same)e(w)o(eigh)o(t)f(v)m(alue\).)21 b(W)l(e)15 b(ha)o(v)o(e)g(brie\015y)h(discussed)h(suc)o(h)e(systems;)f(a)h(more)f (extensiv)o(e)i(in)o(tro)q(duction)g(to)f(this)0 1715 y(area)g(can)g(b)q(e)h(found)f(in)h([Hertz,)f(et)g(al.,)g(1991].)91 1771 y(The)d(implemen)o(tation)h(of)e(neural)i(net)o(w)o(orks)d(in)j (sp)q(ecialist)h(VLSI)e(hardw)o(are)f(has)h(b)q(een)h(the)f(fo)q(cus)g (of)f(m)o(uc)o(h)0 1828 y(researc)o(h,)17 b(although)h(b)o(y)f(far)g (the)g(ma)s(jorit)o(y)f(of)h(w)o(ork)f(in)i(neural)g(computing)g(is)g (undertak)o(en)g(using)g(soft)o(w)o(are)0 1884 y(implemen)o(tations)e (running)h(on)e(standard)g(platforms.)91 1941 y(An)e(implicit)i (assumption)d(throughout)g(most)g(of)g(this)g(c)o(hapter)h(is)g(that)e (the)i(pro)q(cesses)g(whic)o(h)g(giv)o(e)g(rise)g(to)0 1997 y(the)i(data)g(are)g(stationary)f(in)i(time.)k(The)15 b(tec)o(hniques)i(discussed)g(here)e(can)h(readily)g(b)q(e)g(applied)h (to)d(problems)0 2053 y(suc)o(h)21 b(as)f(time)g(series)i(forecasting,) f(pro)o(vided)g(this)g(stationarit)o(y)e(assumption)i(is)g(v)m(alid.)37 b(If,)21 b(ho)o(w)o(ev)o(er,)g(the)0 2110 y(generator)13 b(of)h(the)g(data)g(is)g(itself)h(ev)o(olving)g(with)g(time)f(then)h (more)f(sophisticated)h(tec)o(hniques)g(m)o(ust)f(b)q(e)h(used,)0 2166 y(and)g(these)h(are)f(the)g(fo)q(cus)g(of)g(m)o(uc)o(h)g(curren)o (t)g(researc)o(h)g([see)h(Bengio,)f(1996].)91 2223 y(One)j(of)e(the)i (original)g(motiv)m(ations)f(for)g(neural)g(net)o(w)o(orks)f(w)o(as)h (as)f(mo)q(dels)i(of)f(information)g(pro)q(cessing)0 2279 y(in)g(biological)i(systems)d(suc)o(h)h(as)f(the)h(h)o(uman)f (brain.)25 b(This)17 b(remains)g(the)g(sub)s(ject)g(of)f(considerable)i (researc)o(h)0 2336 y(activit)o(y)l(,)e(and)g(there)g(is)g(a)g(con)o (tin)o(uing)g(\015o)o(w)g(of)f(ideas)h(b)q(et)o(w)o(een)h(the)e (\014elds)i(of)f(neurobiology)g(and)g(of)g(arti\014cial)0 2392 y(neural)g(net)o(w)o(orks.)j(Another)d(historical)g(springb)q (oard)g(for)f(neural)h(net)o(w)o(ork)e(concepts)i(w)o(as)e(that)h(of)g (adaptiv)o(e)0 2449 y(con)o(trol,)g(and)g(again)g(this)h(remains)f(a)g (sub)s(ject)g(of)g(great)f(in)o(terest.)952 2775 y(27)p eop %%Page: 26 4 26 3 bop 0 192 a Fr(Hop\014eld,)15 b(J.)e(J.)g(1982.)18 b(Neural)c(net)o(w)o(orks)e(and)i(ph)o(ysical)g(systems)f(with)h (emergen)o(t)f(collectiv)o(e)i(computational)91 248 y(abilities.)22 b Fq(Pr)n(o)n(c)n(e)n(e)n(dings)14 b(of)i(the)h(National)f(A)n(c)n (ademy)g(of)g(Scienc)n(es)p Fr(,)d(79:2554{2558)o(.)0 342 y(Jacobs,)23 b(R.A.,)f(Jordan,)h(M.I.,)e(No)o(wlan,)i(S.J.,)f(and)g (Hin)o(ton,)h(G.E.)d(1991.)37 b(Adaptiv)o(e)22 b(mixtures)g(of)f(lo)q (cal)91 398 y(exp)q(erts.)f Fq(Neur)n(al)c(Computation)p Fr(,)g(3:79{87.)0 492 y(Jordan,)23 b(M.I.)d(and)i(Jacobs,)g(R.A.)f (1994.)38 b(Hierarc)o(hical)22 b(mixtures)g(of)f(exp)q(erts)h(and)f (the)g(EM)g(algorithm.)91 549 y Fq(Neur)n(al)16 b(Computation)p Fr(,)f(6:181{214.)0 642 y(Le)i(Cun,)g(Y.,)e(Boser,)i(B.,)f(Denk)o(er,)g (J.)g(S.,)g(Henderson,)h(D.,)f(Ho)o(w)o(ard,)f(R.)h(E.,)g(Hubbard,)h (W.)f(and)g(Jac)o(k)o(el,)h(L.)91 699 y(D.)e(1989.)k(Bac)o (kpropagation)c(applied)i(to)e(handwritten)h(zip)g(co)q(de)g (recognition.)21 b Fq(Neur)n(al)c(Computation)p Fr(,)91 755 y(1\(4\):541{551)o(.)0 849 y(Luttrell,)i(S.)f(1989.)27 b(The)18 b(Gibbs)h(mac)o(hine)f(applied)i(to)e(hidden)h(Mark)o(o)o(v)d (mo)q(del)j(problems.)29 b Fq(R)n(oyal)19 b(Signals)91 906 y(and)d(R)n(adar)h(Establishment:)j(SP)15 b(R)n(ese)n(ar)n(ch)h (Note)g(99)p Fr(,)f(Malv)o(ern,)g(UK.)0 999 y(MacKa)o(y)l(,)j(D.)f(J.)h (C.)f(1992.)27 b(A)18 b(practical)h(Ba)o(y)o(esian)f(framew)o(ork)f (for)g(bac)o(k-propagation)g(net)o(w)o(orks.)27 b Fq(Neur)n(al)91 1056 y(Computation)p Fr(,)15 b(4:448{472.)0 1150 y(McCullagh,)h(P)l(.)f (and)g(Nelder,)h(J.A.)f(1983.)k Fq(Gener)n(alize)n(d)14 b(Line)n(ar)i(Mo)n(dels)p Fr(.)i(Chapman)e(and)f(Hall,)h(London.)0 1243 y(Neal,)g(R.)g(M.)f(1994.)21 b Fq(Bayesian)16 b(L)n(e)n(arning)f (for)j(Neur)n(al)e(Networks)p Fr(.)22 b(Unpublished)d(PhD)d(thesis,)g (Departmen)o(t)91 1300 y(of)f(Computer)f(Science,)j(Univ)o(ersit)o(y)f (of)f(T)l(oron)o(to,)e(Canada.)0 1394 y(P)o(earl,)g(J.)g(1988.)19 b Fq(Pr)n(ob)n(abilistic)13 b(R)n(e)n(asoning)g(in)g(Intel)r(ligent)g (Systems)p Fr(.)18 b(Morgan)12 b(Kaufmann,)i(San)f(Mateo,)f(CA.)0 1487 y(P)o(oggio,)j(T.)g(and)g(V)l(etter,)h(T.)f(1992.)20 b Fq(R)n(e)n(c)n(o)n(gnition)15 b(and)h(structur)n(e)h(fr)n(om)g(one)g (2D)g(mo)n(del)f(view:)22 b(Observations)91 1544 y(on)12 b(pr)n(ototyp)n(es,)h(obje)n(ct)f(classes)f(and)h(symmetries)p Fr(.)18 b(AI)11 b(Memo)f(1347,)g(Arti\014cial)j(In)o(telligence)g(Lab)q (oratory)l(,)91 1600 y(Massac)o(h)o(usetts)h(Institute)h(of)g(T)l(ec)o (hnology)l(.)0 1694 y(Rabiner,)24 b(L.R.)d(1989.)37 b(A)22 b(tutorial)f(on)g(Hidden)i(Mark)o(o)o(v)d(mo)q(dels)i(and)f(selected)i (applications)g(in)f(sp)q(eec)o(h)91 1751 y(recognition.)f Fq(Pr)n(o)n(c)n(e)n(e)n(dings)13 b(of)k(the)f(IEEE)p Fr(,)e(77:257{286.)0 1844 y(Rumelhart,)20 b(D.)e(E.,)g(Durbin,)i(R.,)f (Golden,)h(R.,)f(and)g(Chauvin,)h(Y.)e(1995.)29 b(Bac)o(kpropagation:)c (The)19 b(basic)91 1901 y(theory)l(.)31 b(In)19 b Fq(Backpr)n(op)n (agation:)28 b(The)n(ory,)21 b(A)o(r)n(chite)n(ctur)n(es,)e(and)h (Applic)n(ations)p Fr(,)e(ed.)32 b(Y.)18 b(Chauvin,)j(and)91 1957 y(D.E.)14 b(Rumelhart,)i(p.)k(1{35.)e(La)o(wrence)e(Erlbaum,)f (Hillsdale,)j(NJ.)0 2051 y(Saul,)h(L.)f(K.,)h(Jaakk)o(ola,)f(T.,)g(and) g(Jordan,)g(M.)g(I.)g(1995.)27 b(Mean)18 b(\014eld)i(learning)f(theory) f(for)f(sigmoid)i(b)q(elief)91 2108 y(net)o(w)o(orks.)g(Computational)c (Cognitiv)o(e)g(Science)i(T)l(ec)o(h.)k(Rep.)f(9501,)14 b(MIT,)h(Cam)o(bridge,)g(MA.)0 2201 y(Saul,)j(L.)g(K.)f(and)h(Jordan,)g (M.)e(I.)i(1995.)25 b(Boltzmann)18 b(c)o(hains)g(and)g(hidden)h(Mark)o (o)o(v)d(mo)q(dels.)27 b(In)19 b Fq(A)n(dvanc)n(es)91 2258 y(in)i(Neur)n(al)g(Information)h(Pr)n(o)n(c)n(essing)d(Systems)i (7)p Fr(,)h(ed.,)g(G.)e(T)l(esauro,)i(D.)e(T)l(ouretzky)l(,)j(and)e(T.) f(Leen.)91 2314 y(Cam)o(bridge,)15 b(MA,)f(MIT)h(Press.)0 2408 y(Sandler,)21 b(D.)e(G.,)g(Barrett,)g(T.)f(K.,)i(P)o(almer,)g(D.)f (A.,)g(F)l(ugate,)h(R.)f(Q.,)h(and)f(Wild,)i(W.)e(J.)g(1991.)31 b(Use)19 b(of)g(a)91 2465 y(neural)h(net)o(w)o(ork)f(to)g(con)o(trol)g (an)h(adaptiv)o(e)g(optics)g(system)f(for)g(an)h(astronomical)f (telescop)q(e,)j Fq(Natur)n(e)p Fr(,)91 2521 y(351:300{302.)952 2775 y(26)p eop %%Page: 25 5 25 4 bop 0 192 a Fr(is)18 b(in)h(fact)e(p)q(ossible)i(to)e(asso)q (ciate)h(binary)g(hidden)i(v)m(alues)f(with)f(the)f(hidden)j(units)e (of)g(suc)o(h)g(a)f(net)o(w)o(ork)g(\(cf.)0 248 y(our)e(earlier)i (discussion)g(of)e(the)g(logistic)i(function;)f(see)g(also)f([Amari,)g (1995]\))f(and)h(apply)h(the)g(EM)f(algorithm)0 304 y(directly)l(.)20 b(F)l(or)12 b Fm(N)k Fr(hidden)e(units,)f(ho)o(w)o(ev)o(er,)e(there)h (are)g(2)973 288 y Fi(N)1018 304 y Fr(patterns)f(whose)h(probabilities) j(m)o(ust)c(b)q(e)i(calculated)0 361 y(in)18 b(the)f(E)g(step.)26 b(F)l(or)17 b(large)g Fm(N)5 b Fr(,)16 b(this)i(is)g(an)f(in)o (tractable)g(computation,)h(and)f(recen)o(t)g(researc)o(h)g(has)g (therefore)0 417 y(b)q(egun)g(to)d(fo)q(cus)i(on)g(fast)f(metho)q(ds)g (for)g(appro)o(ximating)h(these)g(distributions)h([Hin)o(ton,)e(et)h (al.,)f(1995];)f([Saul,)0 474 y(et)h(al.,)g(1995].)0 616 y Fp(References)0 717 y Fr(Amari,)h(S.)f(1995.)21 b(The)16 b(EM)f(algorithm)h(and)g(information)g(geometry)f(in)i(neural) g(net)o(w)o(ork)d(learning.)23 b Fq(Neur)n(al)91 774 y(Computation)p Fr(,)15 b(7\(1\):13{18.)0 865 y(Anderson,)22 b(T.)d(W.)h(1984.)33 b Fq(A)o(n)20 b(Intr)n(o)n(duction)h(to)g (Multivariate)g(Statistic)n(al)f(A)o(nalysis)p Fr(.)33 b(John)20 b(Wiley)l(,)j(New)91 922 y(Y)l(ork.)0 1013 y(Bengio,)13 b(Y.)e(1996.)18 b Fq(Neur)n(al)13 b(Networks)g(for)h(Sp)n (e)n(e)n(ch)e(and)h(Se)n(quenc)n(e)f(R)n(e)n(c)n(o)n(gnition)p Fr(.)17 b(Thomson)11 b(Computer)h(Press,)91 1070 y(London.)0 1161 y(Bishop,)k(C.)e(M.)h(1995.)k Fq(Neur)n(al)d(Networks)g(for)g (Pattern)h(R)n(e)n(c)n(o)n(gnition)p Fr(.)g(Oxford)f(Univ)o(ersit)o(y)f (Press.)0 1253 y(Breiman,)23 b(L.,)e(F)l(riedman,)i(J.H.,)e(Olshen,)j (R.A.,)d(&)h(Stone,)g(C.J.)e(1984.)35 b Fq(Classi\014c)n(ation)20 b(and)h(R)n(e)n(gr)n(ession)91 1309 y(T)m(r)n(e)n(es)p Fr(.)d(W)l(adsw)o(orth)c(In)o(ternational)i(Group,)e(Belmon)o(t,)h(CA.) 0 1401 y(Bun)o(tine,)f(W.)f(1994.)18 b(Op)q(erations)c(for)e(learning)j (with)e(graphical)h(mo)q(dels.)20 b Fq(Journal)15 b(of)f(A)o (rti\014cial)g(Intel)r(ligenc)n(e)91 1457 y(R)n(ese)n(ar)n(ch)g Fr(2:159{225.)0 1549 y(Dempster,)k(A.P)l(.,)f(Laird,)i(N.M.,)d(and)i (Rubin,)i(D.B.)d(1977.)26 b(Maxim)o(um-lik)o(eliho)q(o)q(d)20 b(from)d(incomplete)j(data)91 1605 y(via)15 b(the)h(EM)e(algorithm.)20 b Fq(J.)c(of)h(R)n(oyal)f(Statistic)n(al)f(So)n(ciety,)h(B39)p Fr(:1{38.)0 1697 y(Fletc)o(her,)f(R.)g(1987.)k Fq(Pr)n(actic)n(al)d (Metho)n(ds)g(of)g(Optimization)p Fr(,)f(2nd)g(ed.)20 b(John)c(Wiley)l(,)g(New)g(Y)l(ork.)0 1789 y(Geman,)e(S.,)g(Bienensto)q (c)o(k,)h(E.,)e(and)i(Doursat,)d(R.)j(1992.)j(Neural)d(net)o(w)o(orks)e (and)h(the)h(bias/v)m(ariance)g(dilema.)91 1845 y Fq(Neur)n(al)h (Computation)p Fr(,)f(4:1{58.)0 1937 y(Hec)o(k)o(erman,)j(D.,)g (Jordan,)g(M.)f(I.,)h(and)g(Sm)o(yth,)g(P)l(.)g(1996.)27 b Fq(Conditional)18 b(indep)n(endenc)n(e)f(gr)n(aphs)i(for)g(hidden)91 1993 y(Markov)h(pr)n(ob)n(ability)f(mo)n(dels)p Fr(.)30 b(T)l(ec)o(h.)g(Rep.,)20 b(Cen)o(ter)e(for)g(Biological)j(and)e (Computational)f(Learning,)91 2050 y(Massac)o(h)o(usetts)c(Institute)h (of)g(T)l(ec)o(hnology)l(.)0 2141 y(Hertz,)h(J.,)g(Krogh,)g(A,)f(and)i (P)o(almer,)e(R.)i(G.)e(1991.)21 b Fq(Intr)n(o)n(duction)c(to)g(the)h (The)n(ory)f(of)g(Neur)n(al)g(Computation)p Fr(.)91 2198 y(Addison-W)l(esley)l(,)g(Redw)o(o)q(o)q(d)f(Cit)o(y)l(,)f(CA.)0 2289 y(Hin)o(ton,)g(G.)g(E.,)f(Da)o(y)o(an,)g(P)l(.,)h(F)l(rey)l(,)g (B.,)f(and)i(Neal,)f(R.)h(1995.)j(The)c(w)o(ak)o(e-sleep)h(algorithm)f (for)g(unsup)q(ervised)91 2346 y(neural)h(net)o(w)o(orks.)j Fq(Scienc)n(e)p Fr(,)13 b(268:1158{116)o(1.)0 2437 y(Hin)o(ton,)g(G.)f (E.)h(and)g(Sejno)o(wski,)g(T.)f(1986.)18 b(Learning)c(and)f (relearning)h(in)g(Boltzmann)f(mac)o(hines.)20 b(In)14 b Fq(Par)n(al)r(lel)91 2494 y(distribute)n(d)g(pr)n(o)n(c)n(essing)p Fr(:)i Fq(V)m(olume)d(1)p Fr(,)g(ed.,)f(D.)f(E.)h(Rumelhart)h(and)g(J.) f(L.)g(McClelland,)i(p.)19 b(282{317.)e(MIT)91 2550 y(Press,)d(Cam)o (bridge,)h(MA.)952 2775 y(25)p eop %%Page: 24 6 24 5 bop 225 150 a 23681433 10419828 3552215 33614479 34074951 47099740 startTexFig 225 150 a %%BeginDocument: ps/junction-tree.ps save userdict /IslandDrawDict 300 dict dup begin put /ncpoint errordict /nocurrentpoint get def errordict begin /nocurrentpoint { dup /pathbbox load eq { pop 0 0 1 1 } { ncpoint } ifelse } bind def end /image_raster { %% sw sh dw dh xs ys translate scale /sh exch def /sw exch def /imagebuf sw 7 add 8 idiv string def sw sh 1 [sw 0 0 sh 0 0] { currentfile imagebuf readhexstring pop } image } bind def /m {moveto} bind def /l {lineto} bind def /c {curveto} bind def /n {newpath} bind def /cl {closepath} bind def /ar { %% sa ea sx sy rot tx ty matrix currentmatrix 8 1 roll translate rotate scale n 0 0 1 5 3 roll arc setmatrix } bind def /arn { %% sa ea sx sy rot tx ty matrix currentmatrix 8 1 roll translate rotate scale n 0 0 1 5 3 roll arcn setmatrix } bind def /el { %% sx sy rot tx ty matrix currentmatrix 6 1 roll translate rotate scale n 0 0 1 0 360 arc setmatrix cl } bind def /bp {setlinejoin setlinewidth setrgbcolor} bind def /bpbw {setlinejoin setlinewidth setgray} bind def /lw {setlinewidth} bind def /lj {setlinejoin} bind def /gr {setgray} bind def /BPSIDE 32 def %% pixels per pattern side /PATFREQ 3.0 def %% pattern pixels per mm /dp_mat [PATFREQ 0 0 PATFREQ 0 0] def /dp_pw BPSIDE def %% pattern pixel width /dp_ph BPSIDE def %% pattern pixel height /dp_w dp_pw PATFREQ div def %% pattern mm width /dp_h dp_ph PATFREQ div def %% pattern mm height /dp_bs 1 def %% pattern bits per pixel /savemat matrix def /topmat matrix def /patmat matrix def /patpath { /inv exch def topmat setmatrix pathbbox %% get lo - hi indecies /hy exch dp_h div floor cvi def /hx exch dp_w div floor cvi def /ly exch dp_h div floor cvi def /lx exch dp_w div floor cvi def lx 1 hx { dp_w mul ly 1 hy { dp_h mul exch dup 3 1 roll exch patmat currentmatrix pop translate dp_pw dp_ph inv dp_mat dp_proc imagemask patmat setmatrix } for pop } for } bind def /setpattern { /blue exch def /green exch def /red exch def /freq exch def /bwidth exch def /bpside exch def /bstring exch def /onbits 0 def /offbits 0 def freq 0 {/y exch def /x exch def /xindex x 1 add 2 div bpside mul cvi def /yindex y 1 add 2 div bpside mul cvi def bstring yindex bwidth mul xindex 8 idiv add get not 1 7 xindex 8 mod sub bitshift and 0 ne {/onbits onbits 1 add def 1} {/offbits offbits 1 add def 0} ifelse } setscreen {} settransfer systemdict /setcmykcolor known { /fact 1 onbits offbits onbits add div sub def 1 red sub fact mul 1 green sub fact mul 1 blue sub fact mul 0 setcmykcolor } { offbits offbits onbits add div setgray } ifelse } bind def /dmatrix matrix def /dpi 72 0 dmatrix defaultmatrix dtransform dup mul exch dup mul add sqrt def /B {gsave bp stroke grestore} bind def %% brush: gr lw lj /Bbw {gsave bpbw stroke grestore} bind def %% brush: gr lw lj /F {gsave setrgbcolor eofill grestore} bind def %% fill: gr /Fbw {gsave setgray eofill grestore} bind def %% fill: gr /PB {gsave setlinejoin setlinewidth setpattern stroke grestore} bind def /PF {gsave eoclip patpath grestore} bind def /BB { gsave setrgbcolor setlinejoin setlinewidth strokepath clip patpath grestore } bind def /BLACK { 0.0 } bind def /CP {closepath} bind def /FI {eofill} bind def /E {exch} bind def /FF {findfont} bind def /GR {grestore} bind def /GS {gsave} bind def /MF {makefont} bind def /NP {newpath} bind def /RO {rotate} bind def /ST {stroke} bind def /SC {scale} bind def /SF {setfont} bind def /SG {setgray} bind def /SLC {setlinecap} bind def /SLJ {setlinejoin} bind def /SLW {setlinewidth} bind def /TR {translate} bind def /WHITE { 1.0 } bind def /m {moveto} bind def /r {rmoveto} bind def /l {lineto} bind def /sp {x 0 rmoveto} bind def /rl {rlineto} bind def /s {show} bind def /box { NP m l l l CP } bind def /pageboundary { NP m l l l CP } bind def /BS { % black stroke GS SLJ SLW BLACK SG ST GR } bind def /WS { % white stroke GS SLJ SLW WHITE SG ST GR } bind def /reencode_small_dict 12 dict def /ReencodeSmall { reencode_small_dict begin /new_codes_and_names E def /new_font_name E def /base_font_name E def /base_font_dict base_font_name FF def /newfont base_font_dict maxlength dict def base_font_dict { E dup /FID ne { dup /Encoding eq { E dup length array copy newfont 3 1 roll put } { E newfont 3 1 roll put } ifelse } { pop pop } ifelse } forall newfont /FontName new_font_name put new_codes_and_names aload pop new_codes_and_names length 2 idiv { newfont /Encoding get 3 1 roll put } repeat new_font_name newfont definefont pop end %reencode_small_dict } def /extended_Zapf [ 8#223 /a89 8#224 /a90 8#225 /a93 8#226 /a94 8#227 /a91 8#230 /a92 8#231 /a205 8#232 /a85 8#233 /a206 8#234 /a86 8#235 /a87 8#236 /a88 8#237 /a95 8#240 /a96 ] def /extended_Standard [ 29 /thorn 30 /yacute 31 /divide 128 /Acircumflex 129 /Adieresis 130 /Agrave 131 /Aring 132 /Atilde 133 /Ccedilla 134 /Eacute 135 /Ecircumflex 136 /Edieresis 137 /Egrave 138 /Iacute 139 /Icircumflex 140 /Idieresis 141 /Igrave 142 /Ntilde 143 /Oacute 144 /Ocircumflex 145 /Odieresis 146 /Ograve 147 /Otilde 148 /Scaron 149 /Uacute 150 /Ucircumflex 151 /Udieresis 152 /Ugrave 153 /Ydieresis 154 /Zcaron 155 /aacute 156 /acircumflex 157 /adieresis 158 /agrave 159 /aring 160 /atilde 161 /exclamdown 162 /cent 163 /sterling 164 /fraction 165 /yen 166 /florin 167 /section 168 /currency 169 /quotesingle 170 /quotedblleft 171 /guillemotleft 172 /guilsinglleft 173 /guilsinglright 174 /fi 175 /fl 176 /plusminus 177 /endash 178 /dagger 179 /daggerdbl 180 /periodcentered 181 /twosuperior 182 /paragraph 183 /bullet 184 /quotesinglebase 185 /quotedblbase 186 /quotedblright 187 /guillemotright 188 /ellipsis 189 /perthousand 190 /threesuperior 191 /questiondown 192 /mu 193 /grave 194 /acute 195 /circumflex 196 /tilde 197 /macron 198 /breve 199 /dotaccent 200 /dieresis 201 /onesuperior 202 /ring 203 /cedilla 204 /onequarter 205 /hungarumlaut 206 /ogonek 207 /caron 208 /emdash 209 /ccedilla 210 /copyright 211 /eacute 212 /ecircumflex 213 /edieresis 214 /egrave 215 /iacute 216 /icircumflex 217 /idieresis 218 /igrave 219 /logicalnot 220 /minus 221 /ntilde 222 /oacute 223 /ocircumflex 224 /odieresis 225 /AE 226 /onehalf 227 /ordfeminine 228 /ograve 229 /otilde 230 /registered 231 /scaron 232 /Lslash 233 /Oslash 234 /OE 235 /ordmasculine 236 /trademark 237 /uacute 238 /ucircumflex 239 /udieresis 240 /ugrave 241 /ae 242 /ydieresis 243 /zcaron 244 /Aacute 245 /dotlessi 246 /threequarters 247 /Eth 248 /lslash 249 /oslash 250 /oe 251 /germandbls 252 /multiply 253 /Yacute 254 /Thorn 255 /eth ] def /extended_Symbol [ ] def /extend_font { % stack: fontname newfontname E dup (ZapfDingbats) eq { cvn E cvn extended_Zapf ReencodeSmall } { dup (Symbol) eq { cvn E cvn extended_Symbol ReencodeSmall } { cvn E cvn extended_Standard ReencodeSmall } ifelse } ifelse } bind def /getfont { /f E def f cvn where { begin f cvn load exec SF end } { f 0 f length 8 sub getinterval dup dup length 1 add string /localfont exch def localfont exch 0 exch putinterval localfont dup length 1 sub (X) putinterval localfont extend_font localfont FF /xsz f f length 4 sub 4 getinterval cvi def /ysz f f length 8 sub 4 getinterval cvi def [ xsz 0 0 ysz neg 0 0 ] MF dup f cvn E def SF } ifelse } bind def /ul { % space drop thickness GS currentpoint currentlinewidth currentpoint NP m 6 -3 roll SLW 0 E r 0 rl ST SLW m GR } bind def /ss { currentpoint pop E m } bind def /image_raster { % sw sh dw dh xs ys TR SC /sh E def /sw E def /imagebuf sw 7 add 8 idiv string def sw sh 1 [sw 0 0 sh 0 0] { currentfile imagebuf readhexstring pop } image } bind def /imagemask_raster { TR SC /sh E def /sw E def /imagebuf sw 7 add 8 idiv string def sw sh false [sw 0 0 sh 0 0] { currentfile imagebuf readhexstring pop } imagemask } bind def /image_color_raster { % sw sh sd dw dh xs ys systemdict /colorimage known not { /colorimage /colimg load def } if TR SC /sd E def /sh E def /sw E def /imagebuf sw 3 mul sd mul 7 add 8 idiv string def sw sh sd [sw 0 0 sh 0 0] { currentfile imagebuf readhexstring pop } false 3 colorimage } bind def /nx { /x E def } bind def 0. nx gsave 2.83465 -2.83465 scale 0 -279.4 translate topmat currentmatrix pop n 169.28 35.444 m 169.28 84.028 l 1 Fbw gsave 0 0.352 0 Bbw grestore 12.375 5.806 0 169.74 50.944 el 1 Fbw gsave 0 0.352 0 Bbw grestore 12.375 5.806 0 169.74 66.444 el 1 Fbw gsave 0 0.352 0 Bbw grestore 12.375 5.806 0 169.74 81.944 el 1 Fbw gsave 0 0.352 0 Bbw grestore 12.375 5.806 0 169.74 35.444 el 1 Fbw gsave 0 0.352 0 Bbw grestore n 139.33 38.194 m 139.33 52.25 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 138.64 38.722 m 124.12 53.236 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 124.33 52.833 m 138.69 52.833 l 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 124.42 52.991 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 139.31 37.797 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 139.31 52.991 el 1 Fbw gsave 0 0.352 0 Bbw grestore n 139.33 67.75 m 139.33 81.806 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 138.64 68.278 m 124.12 82.792 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 124.33 82.389 m 138.69 82.389 l 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 124.42 82.547 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 139.31 67.352 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 139.31 82.547 el 1 Fbw gsave 0 0.352 0 Bbw grestore n 118.26 45.927 m 118.26 31.872 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 118.95 45.399 m 133.47 30.885 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 133.26 31.288 m 118.9 31.288 l 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 133.17 31.13 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 118.28 46.325 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 118.28 31.13 el 1 Fbw gsave 0 0.352 0 Bbw grestore n 118.26 76.01 m 118.26 61.955 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 118.95 75.483 m 133.47 60.969 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 133.26 61.372 m 118.9 61.372 l 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 133.17 61.214 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 118.28 76.408 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 90 118.28 61.214 el 1 Fbw gsave 0 0.352 0 Bbw grestore n savemat currentmatrix pop [1 0 0 1 36.056 88.611] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Helvetica03600360) getfont () s 0.00 SG (\(a\)) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 79.139 88.611] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Helvetica03600360) getfont () s 0.00 SG (\(b\)) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 125.583 99.917] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Helvetica03600360) getfont () s 0.00 SG (\(c\)) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 165.306 99.917] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Helvetica03600360) getfont () s 0.00 SG (\(d\)) s savemat setmatrix n 32.528 70.333 m 46.889 70.333 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 31.778 39.944 m 31.778 71.111 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 46.75 39.944 m 46.75 70.806 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 32.389 39.944 m 46.75 39.944 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 32.306 55.139 m 46.667 55.139 l 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 32.005 39.797 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 32.005 54.991 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 46.894 39.797 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 46.894 54.991 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 32.005 54.991 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 90 32.005 70.186 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 46.894 54.991 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 46.894 70.186 el 1 Fbw gsave 0 0.352 0 Bbw grestore n 90.444 39.722 m 75.319 54.847 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 90.444 55.306 m 75.319 70.431 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 76.139 70.333 m 90.5 70.333 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 75.389 39.944 m 75.389 71.111 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 90.361 39.944 m 90.361 70.806 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 76 39.944 m 90.361 39.944 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 75.917 55.139 m 90.278 55.139 l 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 75.616 39.797 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 75.616 54.991 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 90.505 39.797 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 90.505 54.991 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 75.616 54.991 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 90 75.616 70.186 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 90.505 54.991 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.203 3.203 0 90.505 70.186 el 1 Fbw gsave 0 0.352 0 Bbw grestore n savemat currentmatrix pop [1 0 0 1 21.389 41.778] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 25.361 43] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (i) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 52.167 41.778] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 55.917 42.778] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (j) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 21.222 56.667] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 25.056 58.056] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (k) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 52 56.667] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 55.611 58.444] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (l) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 21.056 72.083] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 24.444 73.028] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (m) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 51.528 72.083] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 55.306 73.111] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (n) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 160.333 36.722] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 164.306 37.944] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (i) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 166.361 36.722] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 170.111 37.722] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (j) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 172.083 36.722] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 175.917 38.111] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (k) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 160.333 52.917] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 164.083 53.917] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (j) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 166.056 52.917] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 169.889 54.306] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (k) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 172.778 52.917] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 176.389 54.694] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (l) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 167.194 68.5] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 170.806 70.278] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (l) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 160.333 68.5] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 164.167 69.889] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (k) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 173 68.5] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 176.389 69.444] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (m) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 165.833 83.472] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 169.222 84.417] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (m) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 160.333 83.472] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 163.944 85.25] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (l) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 173.167 83.472] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 176.944 84.5] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (n) s savemat setmatrix userdict /#copies 1 put showpage grestore end restore %%EndDocument endTexFig 0 908 a Fr(Figure)12 b(7:)18 b(The)12 b(basic)h(structure)e(of)g(the)h (junction)h(tree)f(algorithm)g(for)f(undirected)i(graphs.)19 b(The)12 b(graph)f(in)i(\(a\))0 964 y(is)j(\014rst)f(triangulated)h (\(b\),)f(then)h(the)f(cliques)j(are)d(iden)o(ti\014ed)i(\(c\),)e(and)h (arranged)f(in)o(to)g(a)g(tree)h(\(d\).)k(Pro)q(ducts)0 1021 y(of)15 b(p)q(oten)o(tial)h(functions)g(on)f(the)g(no)q(des)h(in)g (\(d\))f(yield)h(probabilit)o(y)h(distributions)g(on)e(the)g(no)q(des)h (in)g(\(a\).)0 1157 y(case)i(is)g(the)g(state)f(estimation)i(algorithm) e(of)h(the)g(Kalman)g(\014lter)h([Shac)o(h)o(ter)e(and)h(Kenley)l(,)i (1989].)27 b(Finally)l(,)0 1213 y(there)20 b(are)f(a)g(v)m(ariet)o(y)h (of)f(sp)q(ecial)i(cases)f(of)f(the)g(Boltzmann)h(mac)o(hine)h(whic)o (h)f(are)f(amenable)i(to)e(the)g(exact)0 1270 y(calculations)e(of)d (the)i(junction)g(tree)f(algorithm)g([Saul)h(and)f(Jordan,)g(1995].)91 1326 y(F)l(or)i(graphs)g(that)g(are)g(outside)h(of)f(the)h(tractable)f (categories)h(of)f(trees)g(and)h(c)o(hains,)g(the)g(junction)g(tree)0 1383 y(algorithm)f(often)f(p)q(erforms)h(surprisingly)h(w)o(ell,)g(but) f(for)f(highly)j(connected)e(graphs)g(the)g(algorithm)f(can)h(b)q(e)0 1439 y(to)q(o)f(slo)o(w.)23 b(In)17 b(suc)o(h)g(cases,)f(appro)o (ximate)g(algorithms)g(suc)o(h)h(as)f(Gibbs)h(sampling)g(are)f (utilized.)26 b(A)17 b(virtue)f(of)0 1496 y(the)d(graphical)i(framew)o (ork)c(is)j(that)f(Gibbs)h(sampling)g(has)f(a)g(generic)h(form,)f(whic) o(h)h(is)g(based)f(on)h(the)f(notion)g(of)0 1552 y(a)j Fq(Markov)h(b)n(oundary)f Fr([P)o(earl,)g(1988].)21 b(A)16 b(sp)q(ecial)i(case)e(of)f(this)i(generic)g(form)e(is)h(the)h(sto)q(c)o (hastic)f(up)q(date)g(rule)0 1608 y(for)f(general)g(Boltzmann)h(mac)o (hines.)91 1665 y(Our)d(discussion)h(has)e(emphasized)i(the)f(unifying) h(framew)o(ork)d(of)h(graphical)h(mo)q(dels)h(b)q(oth)e(for)g (expressing)0 1721 y(probabilistic)24 b(dep)q(endencies)g(in)f(graphs)e (and)g(for)g(describing)i(algorithms)f(that)f(p)q(erform)g(the)g (inferen)o(tial)0 1778 y(step)c(of)f(calculating)i(p)q(osterior)f (probabilities)i(on)d(these)h(graphs.)24 b(The)17 b(uni\014cation)h(go) q(es)f(further,)f(ho)o(w)o(ev)o(er,)0 1834 y(when)f(w)o(e)f(consider)h (learning.)21 b(A)15 b(generic)g(metho)q(dology)g(kno)o(wn)f(as)g(the)g (Exp)q(ectation-Maximization)i(\(EM\))0 1891 y(algorithm)d(is)h(a)o(v)m (ailable)h(for)d(MAP)h(and)g(Ba)o(y)o(esian)h(estimation)f(in)h (graphical)g(mo)q(dels)g([Dempster,)f(Laird,)h(and)0 1947 y(Rubin,)22 b(1977].)32 b(EM)19 b(is)h(an)f(iterativ)o(e)h(metho)q (d,)h(based)f(on)f(t)o(w)o(o)f(alternating)i(steps:)29 b(an)19 b Fq(E)i(step)p Fr(,)f(in)g(whic)o(h)0 2004 y(the)f(v)m(alues)g (of)f(hidden)j(v)m(ariables)f(are)e(estimated,)h(based)g(on)f(the)h (curren)o(t)g(v)m(alues)g(of)f(the)h(parameters)f(and)0 2060 y(the)k(v)m(alues)g(of)f(visible)j(v)m(ariables,)g(and)e(an)f Fq(M)h(step)p Fr(,)h(in)f(whic)o(h)g(the)g(parameters)f(are)g(up)q (dated)h(based)g(on)0 2117 y(the)c(estimated)f(v)m(alues)i(obtained)f (from)e(the)i(E)f(step.)27 b(Within)18 b(the)g(framew)o(ork)e(of)h(the) g(EM)g(algorithm,)h(the)0 2173 y(junction)g(tree)g(algorithm)f(can)h (readily)h(b)q(e)f(view)o(ed)g(as)f(pro)o(viding)i(a)e(generic)h(E)g (step.)27 b(Moreo)o(v)o(er,)16 b(once)i(the)0 2229 y(estimated)h(v)m (alues)g(of)f(the)g(hidden)j(no)q(des)e(are)f(obtained)h(from)e(the)i (E)f(step,)h(the)f(graph)h(can)f(b)q(e)h(view)o(ed)g(as)0 2286 y(fully)c(observ)o(ed,)e(and)h(the)f(M)g(step)g(is)h(a)f(standard) g(MAP)g(or)g(ML)h(problem.)19 b(The)14 b(standard)f(algorithms)g(for)g (all)0 2342 y(of)i(the)h(tractable)g(arc)o(hitectures)f(describ)q(ed)j (ab)q(o)o(v)o(e)d(\(mixtures,)h(trees)f(and)h(c)o(hains\))g(are)f(in)i (fact)e(instances)h(of)0 2399 y(this)h(general)f(graphical)i(EM)d (algorithm,)i(and)f(the)g(learning)i(algorithm)e(for)g(general)g (Boltzmann)h(mac)o(hines)0 2455 y(is)f(a)f(sp)q(ecial)i(case)e(of)g(a)f (generalization)j(of)e(EM)f(kno)o(wn)h(as)g(GEM)g([Dempster,)f(et)h (al.,)g(1977].)91 2512 y(What)j(ab)q(out)h(the)g(case)g(of)g(feedforw)o (ard)f(neural)i(net)o(w)o(orks)e(suc)o(h)i(as)e(the)h(m)o(ultila)o(y)o (er)h(p)q(erceptron?)33 b(It)952 2775 y(24)p eop %%Page: 23 7 23 6 bop 0 192 a Fr(v)m(ariables)15 b Fm(O)222 199 y Fi(i)250 192 y Fr(are)e(generally)i(treated)f(as)f(visible,)j(and)e(it) h(is)f(desired)h(to)e(calculate)j(a)d(probabilit)o(y)i(distribution)0 248 y(on)e(the)h(hidden)h(states)d Fm(H)450 255 y Fi(i)464 248 y Fr(.)19 b(A)14 b(similar)g(inferen)o(tial)h(calculation)g(is)f (required)h(in)f(the)f(mixture)h(mo)q(dels)g(and)g(the)0 304 y(Boltzmann)i(mac)o(hine.)91 361 y(Generic)h(algorithms)f(ha)o(v)o (e)f(b)q(een)j(dev)o(elop)q(ed)f(to)f(solv)o(e)g(the)g(inferen)o(tial)h (problem)g(of)f(the)g(calculation)h(of)0 417 y(p)q(osterior)e (probabilities)j(in)e(graphs.)21 b(Although)16 b(a)f(v)m(ariet)o(y)h (of)f(inference)i(algorithms)e(ha)o(v)o(e)g(b)q(een)i(dev)o(elop)q(ed,) 0 474 y(they)22 b(can)g(all)h(b)q(e)g(view)o(ed)g(as)e(essen)o(tially)j (the)e(same)g(underlying)i(algorithm)e([Shac)o(h)o(ter,)g(Andersen,)j (and)0 530 y(Szolo)o(vits,)20 b(1994].)30 b(Let)19 b(us)h(consider)g (undirected)g(graphs.)31 b(A)20 b(sp)q(ecial)g(case)g(of)e(an)h (undirected)i(graph)e(is)g(a)0 587 y Fq(triangulate)n(d)f(gr)n(aph)j Fr([Spiegelhalter,)f(et)d(al.,)g(1993],)f(in)i(whic)o(h)g(an)o(y)f (cycle)h(ha)o(ving)g(four)f(or)f(more)h(no)q(des)h(has)0 643 y(a)j(c)o(hord.)36 b(F)l(or)21 b(example,)h(the)f(graph)g(in)h (Figure)f(5\(a\))f(is)h(not)f(triangulated,)j(but)e(b)q(ecomes)g (triangulated)0 700 y(when)16 b(a)g(link)h(is)f(added)h(b)q(et)o(w)o (een)f(no)q(des)h Fm(X)772 707 y Fi(i)801 700 y Fr(and)f Fm(X)928 707 y Fi(j)946 700 y Fr(.)22 b(In)16 b(a)g(triangulated)g (graph,)f(the)h(cliques)i(of)d(the)h(graph)0 756 y(can)22 b(b)q(e)h(arranged)e(in)i(the)f(form)g(of)f(a)h Fq(junction)g(tr)n(e)n (e)p Fr(,)h(whic)o(h)g(is)f(a)g(tree)g(ha)o(ving)g(the)g(prop)q(ert)o (y)g(that)f(an)o(y)0 813 y(no)q(de)g(that)f(app)q(ears)g(in)i(t)o(w)o (o)d(di\013eren)o(t)i(cliques)h(in)f(the)g(tree)f(also)h(app)q(ears)f (in)h(ev)o(ery)g(clique)h(on)f(the)f(path)0 869 y(that)c(links)h(the)f (t)o(w)o(o)f(cliques)j(\(the)e(\\running)h(in)o(tersection)h(prop)q (ert)o(y"\).)k(This)16 b(cannot)h(b)q(e)f(ac)o(hiev)o(ed)i(in)f(non-)0 925 y(triangulated)j(graphs.)32 b(F)l(or)19 b(example,)i(the)f(cliques) h(in)g(Figure)e(5\(a\))g(are)g Fk(f)p Fm(X)1403 932 y Fi(i)1416 925 y Fm(;)8 b(X)1475 932 y Fi(k)1496 925 y Fk(g)p Fr(,)19 b Fk(f)p Fm(X)1612 932 y Fi(k)1633 925 y Fm(;)8 b(X)1692 932 y Fi(j)1709 925 y Fk(g)p Fr(,)20 b Fk(f)p Fm(X)1826 932 y Fi(j)1844 925 y Fm(;)8 b(X)1903 932 y Fi(l)1915 925 y Fk(g)p Fr(,)0 982 y(and)22 b(it)h(is)f(not)g(p)q (ossible)i(to)e(arrange)f(these)i(cliques)h(in)o(to)e(a)g(tree)g(that)f (ob)q(eys)i(the)f(running)h(in)o(tersection)0 1038 y(prop)q(ert)o(y)l (.)30 b(If)19 b(a)f(c)o(hord)h(is)g(added)g(the)g(resulting)g(cliques)i (are)d Fk(f)p Fm(X)1179 1045 y Fi(i)1192 1038 y Fm(;)8 b(X)1251 1045 y Fi(j)1269 1038 y Fm(;)g(X)1328 1045 y Fi(k)1348 1038 y Fk(g)18 b Fr(and)h Fk(f)p Fm(X)1542 1045 y Fi(i)1555 1038 y Fm(;)8 b(X)1614 1045 y Fi(j)1631 1038 y Fm(;)g(X)1690 1045 y Fi(l)1702 1038 y Fk(g)p Fr(,)19 b(and)g(these)0 1095 y(cliques)h(can)f(b)q(e)g(arranged)e(as)h(a)g (simple)i(c)o(hain)f(that)f(trivially)i(ob)q(eys)e(the)h(running)g(in)o (tersection)g(prop)q(ert)o(y)l(.)0 1151 y(In)14 b(general,)h(it)f (turns)g(out)f(that)g(the)h(probabilit)o(y)h(distributions)h(corresp)q (onding)f(to)e(triangulated)h(graphs)g(can)0 1208 y(b)q(e)h(c)o (haracterized)g(as)e Fq(de)n(c)n(omp)n(osable)p Fr(,)g(whic)o(h)i (implies)i(that)c(they)h(can)g(b)q(e)h(factorized)g(in)o(to)f(a)g(pro)q (duct)g(of)g(lo)q(cal)0 1264 y(functions)i(\(\\p)q(oten)o(tials"\))f (asso)q(ciated)g(with)h(the)g(cliques)h(in)f(the)f(triangulated)h (graph.)1550 1248 y Fj(1)1590 1264 y Fr(The)f(calculation)i(of)0 1321 y(p)q(osterior)e(probabilities)i(in)f(decomp)q(osable)g (distributions)h(is)e(straigh)o(tforw)o(ard,)e(and)i(can)g(b)q(e)h(ac)o (hiev)o(ed)g(via)f(a)0 1377 y(lo)q(cal)h(message-passing)g(algorithm)f (on)g(the)g(junction)h(tree)f([Spiegelhalter,)i(et)e(al.,)g(1993].)91 1434 y(Graphs)j(that)f(are)h(not)f(triangulated)i(can)f(b)q(e)h(turned) g(in)o(to)f(triangulated)g(graphs)g(b)o(y)g(the)g(addition)h(of)0 1490 y(links.)36 b(If)21 b(the)f(p)q(oten)o(tials)h(on)f(the)g(new)g (graph)g(are)g(de\014ned)i(suitably)f(as)f(pro)q(ducts)g(of)g(p)q(oten) o(tials)h(on)f(the)0 1546 y(original)d(graph,)e(then)h(the)g(indep)q (endenci)q(es)j(in)e(the)f(original)g(graph)g(are)f(preserv)o(ed.)22 b(This)17 b(implies)h(that)d(the)0 1603 y(algorithms)g(for)g (triangulated)g(graphs)g(can)h(b)q(e)f(used)h(for)f Fq(al)r(l)g Fr(undirected)i(graphs;)d(an)h(un)o(triangulated)h(graph)0 1659 y(is)e(\014rst)f(triangulated)h(\(see)g(Figure)g(7\).)k(Moreo)o(v) o(er,)12 b(it)i(is)g(p)q(ossible)i(to)c(con)o(v)o(ert)h Fq(dir)n(e)n(cte)n(d)g Fr(graphs)g(to)g(undirected)0 1716 y(graphs)j(in)h(a)g(manner)f(that)g(preserv)o(es)h(the)f (probabilistic)j(structure)d(of)g(the)h(original)g(graph)g ([Spiegelhalter,)0 1772 y(et)f(al.,)h(1993].)22 b(This)c(implies)g (that)e(the)h(junction)g(tree)g(algorithm)f(is)h(indeed)h(generic;)g (it)f(can)g(b)q(e)g(applied)i(to)0 1829 y(an)o(y)c(graphical)h(mo)q (del.)91 1885 y(The)d(problem)h(of)e(calculating)j(p)q(osterior)e (probabilities)i(on)e(graphs)g(is)h(NP-hard;)f(th)o(us,)g(a)g(ma)s(jor) e(issue)j(in)0 1942 y(the)i(use)h(of)f(the)g(inference)i(algorithms)e (is)g(the)h(iden)o(ti\014cation)h(of)d(cases)i(in)g(whic)o(h)g(they)f (are)g(e\016cien)o(t.)23 b(Chain)0 1998 y(structures)d(suc)o(h)h(as)f (HMM's)g(yield)i(e\016cien)o(t)f(algorithms,)g(and)g(indeed)h(the)f (classical)h(forw)o(ard-bac)o(kw)o(ard)0 2055 y(algorithm)14 b(for)g(HMM's)f(is)h(a)g(sp)q(ecial,)i(e\016cien)o(t)f(case)f(of)g(the) g(junction)h(tree)f(algorithm)g([Hec)o(k)o(erman,)f(Jordan,)0 2111 y(and)18 b(Sm)o(yth,)f(1996].)26 b(Decision)18 b(tree)g (structures)f(suc)o(h)h(as)f(the)h(hierarc)o(hical)h(mixture)f(of)f (exp)q(erts)h(yield)h(e\016-)0 2167 y(cien)o(t)f(algorithms,)f(and)g (the)g(recursiv)o(e)h(p)q(osterior)f(probabilit)o(y)i(calculation)f(of) f([Jordan)g(and)g(Jacobs,)g(1994])0 2224 y(describ)q(ed)j(earlier)f(is) f(also)g(a)f(sp)q(ecial)j(case)e(of)f(the)h(junction)h(tree)e (algorithm.)28 b(All)19 b(of)f(the)f(simpler)j(mixture)0 2280 y(mo)q(del)g(calculations)h(describ)q(ed)g(earlier)g(are)e (therefore)g(also)g(sp)q(ecial)i(cases.)33 b(Another)19 b(in)o(teresting)h(sp)q(ecial)p 0 2320 780 2 v 52 2347 a Fc(1)69 2363 y Fb(An)11 b(in)o(teresting)j(example)f(is)f(a)f (Boltzmann)i(mac)o(hine)g(on)e(a)h(triangulated)i(graph.)j(The)11 b(p)q(oten)o(tials)j(are)d(pro)q(ducts)i(of)e(exp)q(\()p Fa(J)1908 2367 y Fd(ij)1935 2363 y Fb(\))0 2408 y(factors,)g(where)h (the)f(pro)q(duct)h(is)g(tak)o(en)g(o)o(v)o(er)f(all)i(\()p Fa(i;)6 b(j)r Fb(\))11 b(pairs)h(in)g(a)g(particular)h(clique.)18 b(Giv)o(en)13 b(that)e(the)g(pro)q(duct)i(across)f(p)q(oten)o(tials)0 2454 y(m)o(ust)i(b)q(e)g(the)g(join)o(t)g(probabilit)o(y)m(,)j(this)d (implies)i(that)e(the)g(partition)i(function)f(\(the)f(denominator)h (of)f(Eq.)f(35\))h(m)o(ust)g(b)q(e)g(unit)o(y)h(in)0 2500 y(this)f(case.)952 2775 y Fr(23)p eop %%Page: 22 8 22 7 bop 225 150 a 23681433 8998941 3815342 31246336 32890880 42363453 startTexFig 225 150 a %%BeginDocument: ps/hmm.ps save userdict /IslandDrawDict 300 dict dup begin put /ncpoint errordict /nocurrentpoint get def errordict begin /nocurrentpoint { dup /pathbbox load eq { pop 0 0 1 1 } { ncpoint } ifelse } bind def end /image_raster { %% sw sh dw dh xs ys translate scale /sh exch def /sw exch def /imagebuf sw 7 add 8 idiv string def sw sh 1 [sw 0 0 sh 0 0] { currentfile imagebuf readhexstring pop } image } bind def /m {moveto} bind def /l {lineto} bind def /c {curveto} bind def /n {newpath} bind def /cl {closepath} bind def /ar { %% sa ea sx sy rot tx ty matrix currentmatrix 8 1 roll translate rotate scale n 0 0 1 5 3 roll arc setmatrix } bind def /arn { %% sa ea sx sy rot tx ty matrix currentmatrix 8 1 roll translate rotate scale n 0 0 1 5 3 roll arcn setmatrix } bind def /el { %% sx sy rot tx ty matrix currentmatrix 6 1 roll translate rotate scale n 0 0 1 0 360 arc setmatrix cl } bind def /bp {setlinejoin setlinewidth setrgbcolor} bind def /bpbw {setlinejoin setlinewidth setgray} bind def /lw {setlinewidth} bind def /lj {setlinejoin} bind def /gr {setgray} bind def /BPSIDE 32 def %% pixels per pattern side /PATFREQ 3.0 def %% pattern pixels per mm /dp_mat [PATFREQ 0 0 PATFREQ 0 0] def /dp_pw BPSIDE def %% pattern pixel width /dp_ph BPSIDE def %% pattern pixel height /dp_w dp_pw PATFREQ div def %% pattern mm width /dp_h dp_ph PATFREQ div def %% pattern mm height /dp_bs 1 def %% pattern bits per pixel /savemat matrix def /topmat matrix def /patmat matrix def /patpath { /inv exch def topmat setmatrix pathbbox %% get lo - hi indecies /hy exch dp_h div floor cvi def /hx exch dp_w div floor cvi def /ly exch dp_h div floor cvi def /lx exch dp_w div floor cvi def lx 1 hx { dp_w mul ly 1 hy { dp_h mul exch dup 3 1 roll exch patmat currentmatrix pop translate dp_pw dp_ph inv dp_mat dp_proc imagemask patmat setmatrix } for pop } for } bind def /setpattern { /blue exch def /green exch def /red exch def /freq exch def /bwidth exch def /bpside exch def /bstring exch def /onbits 0 def /offbits 0 def freq 0 {/y exch def /x exch def /xindex x 1 add 2 div bpside mul cvi def /yindex y 1 add 2 div bpside mul cvi def bstring yindex bwidth mul xindex 8 idiv add get not 1 7 xindex 8 mod sub bitshift and 0 ne {/onbits onbits 1 add def 1} {/offbits offbits 1 add def 0} ifelse } setscreen {} settransfer systemdict /setcmykcolor known { /fact 1 onbits offbits onbits add div sub def 1 red sub fact mul 1 green sub fact mul 1 blue sub fact mul 0 setcmykcolor } { offbits offbits onbits add div setgray } ifelse } bind def /dmatrix matrix def /dpi 72 0 dmatrix defaultmatrix dtransform dup mul exch dup mul add sqrt def /B {gsave bp stroke grestore} bind def %% brush: gr lw lj /Bbw {gsave bpbw stroke grestore} bind def %% brush: gr lw lj /F {gsave setrgbcolor eofill grestore} bind def %% fill: gr /Fbw {gsave setgray eofill grestore} bind def %% fill: gr /PB {gsave setlinejoin setlinewidth setpattern stroke grestore} bind def /PF {gsave eoclip patpath grestore} bind def /BB { gsave setrgbcolor setlinejoin setlinewidth strokepath clip patpath grestore } bind def /BLACK { 0.0 } bind def /CP {closepath} bind def /FI {eofill} bind def /E {exch} bind def /FF {findfont} bind def /GR {grestore} bind def /GS {gsave} bind def /MF {makefont} bind def /NP {newpath} bind def /RO {rotate} bind def /ST {stroke} bind def /SC {scale} bind def /SF {setfont} bind def /SG {setgray} bind def /SLC {setlinecap} bind def /SLJ {setlinejoin} bind def /SLW {setlinewidth} bind def /TR {translate} bind def /WHITE { 1.0 } bind def /m {moveto} bind def /r {rmoveto} bind def /l {lineto} bind def /sp {x 0 rmoveto} bind def /rl {rlineto} bind def /s {show} bind def /box { NP m l l l CP } bind def /pageboundary { NP m l l l CP } bind def /BS { % black stroke GS SLJ SLW BLACK SG ST GR } bind def /WS { % white stroke GS SLJ SLW WHITE SG ST GR } bind def /reencode_small_dict 12 dict def /ReencodeSmall { reencode_small_dict begin /new_codes_and_names E def /new_font_name E def /base_font_name E def /base_font_dict base_font_name FF def /newfont base_font_dict maxlength dict def base_font_dict { E dup /FID ne { dup /Encoding eq { E dup length array copy newfont 3 1 roll put } { E newfont 3 1 roll put } ifelse } { pop pop } ifelse } forall newfont /FontName new_font_name put new_codes_and_names aload pop new_codes_and_names length 2 idiv { newfont /Encoding get 3 1 roll put } repeat new_font_name newfont definefont pop end %reencode_small_dict } def /extended_Zapf [ 8#223 /a89 8#224 /a90 8#225 /a93 8#226 /a94 8#227 /a91 8#230 /a92 8#231 /a205 8#232 /a85 8#233 /a206 8#234 /a86 8#235 /a87 8#236 /a88 8#237 /a95 8#240 /a96 ] def /extended_Standard [ 29 /thorn 30 /yacute 31 /divide 128 /Acircumflex 129 /Adieresis 130 /Agrave 131 /Aring 132 /Atilde 133 /Ccedilla 134 /Eacute 135 /Ecircumflex 136 /Edieresis 137 /Egrave 138 /Iacute 139 /Icircumflex 140 /Idieresis 141 /Igrave 142 /Ntilde 143 /Oacute 144 /Ocircumflex 145 /Odieresis 146 /Ograve 147 /Otilde 148 /Scaron 149 /Uacute 150 /Ucircumflex 151 /Udieresis 152 /Ugrave 153 /Ydieresis 154 /Zcaron 155 /aacute 156 /acircumflex 157 /adieresis 158 /agrave 159 /aring 160 /atilde 161 /exclamdown 162 /cent 163 /sterling 164 /fraction 165 /yen 166 /florin 167 /section 168 /currency 169 /quotesingle 170 /quotedblleft 171 /guillemotleft 172 /guilsinglleft 173 /guilsinglright 174 /fi 175 /fl 176 /plusminus 177 /endash 178 /dagger 179 /daggerdbl 180 /periodcentered 181 /twosuperior 182 /paragraph 183 /bullet 184 /quotesinglebase 185 /quotedblbase 186 /quotedblright 187 /guillemotright 188 /ellipsis 189 /perthousand 190 /threesuperior 191 /questiondown 192 /mu 193 /grave 194 /acute 195 /circumflex 196 /tilde 197 /macron 198 /breve 199 /dotaccent 200 /dieresis 201 /onesuperior 202 /ring 203 /cedilla 204 /onequarter 205 /hungarumlaut 206 /ogonek 207 /caron 208 /emdash 209 /ccedilla 210 /copyright 211 /eacute 212 /ecircumflex 213 /edieresis 214 /egrave 215 /iacute 216 /icircumflex 217 /idieresis 218 /igrave 219 /logicalnot 220 /minus 221 /ntilde 222 /oacute 223 /ocircumflex 224 /odieresis 225 /AE 226 /onehalf 227 /ordfeminine 228 /ograve 229 /otilde 230 /registered 231 /scaron 232 /Lslash 233 /Oslash 234 /OE 235 /ordmasculine 236 /trademark 237 /uacute 238 /ucircumflex 239 /udieresis 240 /ugrave 241 /ae 242 /ydieresis 243 /zcaron 244 /Aacute 245 /dotlessi 246 /threequarters 247 /Eth 248 /lslash 249 /oslash 250 /oe 251 /germandbls 252 /multiply 253 /Yacute 254 /Thorn 255 /eth ] def /extended_Symbol [ ] def /extend_font { % stack: fontname newfontname E dup (ZapfDingbats) eq { cvn E cvn extended_Zapf ReencodeSmall } { dup (Symbol) eq { cvn E cvn extended_Symbol ReencodeSmall } { cvn E cvn extended_Standard ReencodeSmall } ifelse } ifelse } bind def /getfont { /f E def f cvn where { begin f cvn load exec SF end } { f 0 f length 8 sub getinterval dup dup length 1 add string /localfont exch def localfont exch 0 exch putinterval localfont dup length 1 sub (X) putinterval localfont extend_font localfont FF /xsz f f length 4 sub 4 getinterval cvi def /ysz f f length 8 sub 4 getinterval cvi def [ xsz 0 0 ysz neg 0 0 ] MF dup f cvn E def SF } ifelse } bind def /ul { % space drop thickness GS currentpoint currentlinewidth currentpoint NP m 6 -3 roll SLW 0 E r 0 rl ST SLW m GR } bind def /ss { currentpoint pop E m } bind def /image_raster { % sw sh dw dh xs ys TR SC /sh E def /sw E def /imagebuf sw 7 add 8 idiv string def sw sh 1 [sw 0 0 sh 0 0] { currentfile imagebuf readhexstring pop } image } bind def /imagemask_raster { TR SC /sh E def /sw E def /imagebuf sw 7 add 8 idiv string def sw sh false [sw 0 0 sh 0 0] { currentfile imagebuf readhexstring pop } imagemask } bind def /image_color_raster { % sw sh sd dw dh xs ys systemdict /colorimage known not { /colorimage /colimg load def } if TR SC /sd E def /sh E def /sw E def /imagebuf sw 3 mul sd mul 7 add 8 idiv string def sw sh sd [sw 0 0 sh 0 0] { currentfile imagebuf readhexstring pop } false 3 colorimage } bind def /nx { /x E def } bind def 0. nx gsave 2.83465 -2.83465 scale 0 -279.4 translate topmat currentmatrix pop n 115.81 65.083 m 173.86 65.083 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 116.42 66 m 116.42 85.25 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 135.06 65.389 m 135.06 84.639 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 154 65.694 m 154 86.778 l 1 Fbw gsave 0 0.352 0 Bbw grestore n 172.94 66 m 172.94 86.778 l 1 Fbw gsave 0 0.352 0 Bbw grestore 3.58 3.58 90 116.42 65.292 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.58 3.58 0 135.28 65.292 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.58 3.58 0 154.14 65.292 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.58 3.58 0 173 65.292 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.58 3.58 0 116.42 85.986 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.58 3.58 0 135.28 85.986 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.58 3.58 0 154.14 85.986 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.58 3.58 0 173 85.986 el 1 Fbw gsave 0 0.352 0 Bbw grestore n 23.444 66.917 m 23.444 82.5 l 1 Fbw n 23.444 82.5 m 22.259 78.944 l 24.629 78.944 l cl 0 Fbw n 23.444 66.917 m 23.444 78.944 l gsave 0 0.352 0 Bbw grestore n 42.306 66.834 m 42.306 82.417 l 1 Fbw n 42.306 82.417 m 41.121 78.861 l 43.491 78.861 l cl 0 Fbw n 42.306 66.834 m 42.306 78.861 l gsave 0 0.352 0 Bbw grestore n 61.167 66.75 m 61.167 82.334 l 1 Fbw n 61.167 82.334 m 59.982 78.778 l 62.352 78.778 l cl 0 Fbw n 61.167 66.75 m 61.167 78.778 l gsave 0 0.352 0 Bbw grestore n 80.028 66.667 m 80.028 82.25 l 1 Fbw n 80.028 82.25 m 78.843 78.694 l 81.213 78.694 l cl 0 Fbw n 80.028 66.667 m 80.028 78.694 l gsave 0 0.352 0 Bbw grestore n 23.833 65.695 m 38.806 65.695 l 1 Fbw n 38.806 65.695 m 35.25 66.88 l 35.25 64.509 l cl 0 Fbw n 23.833 65.695 m 35.25 65.695 l gsave 0 0.352 0 Bbw grestore n 42.389 65.612 m 57.361 65.612 l 1 Fbw n 57.361 65.612 m 53.805 66.797 l 53.805 64.426 l cl 0 Fbw n 42.389 65.612 m 53.805 65.612 l gsave 0 0.352 0 Bbw grestore n 61.25 65.528 m 76.222 65.528 l 1 Fbw n 76.222 65.528 m 72.666 66.713 l 72.666 64.342 l cl 0 Fbw n 61.25 65.528 m 72.666 65.528 l gsave 0 0.352 0 Bbw grestore 3.58 3.58 0 23.444 65.598 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.58 3.58 0 42.306 65.598 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.58 3.58 0 61.167 65.598 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.58 3.58 0 80.028 65.598 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.58 3.58 0 23.444 86.292 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.58 3.58 0 42.306 86.292 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.58 3.58 0 61.167 86.292 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.58 3.58 0 80.028 86.292 el 1 Fbw gsave 0 0.352 0 Bbw grestore n savemat currentmatrix pop [1 0 0 1 141.472 111.528] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Helvetica03600360) getfont () s 0.00 SG (\(b\)) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 47.667 111.528] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Helvetica03600360) getfont () s 0.00 SG (\(a\)) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 21.0833 96.8611] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (O) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 24.6667 98.6111] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (1) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 39.6389 96.8611] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (O) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 43.6111 98.6944] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (2) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 58.5 96.8611] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (O) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 62.5556 98.4722] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (3) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 77.3611 96.8611] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (O) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 81.1944 98.5556] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (4) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 24.75 59.8889] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (1) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 21.0833 58.0556] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (H) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 62.6389 59.75] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (3) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 58.8056 58.0556] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (H) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 81.5833 59.8333] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (4) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 77.6667 58.0556] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (H) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 43.9167 59.6667] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (2) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 39.8611 58.0556] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (H) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 117.556 59.8889] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (1) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 113.889 58.0556] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (H) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 155.444 59.75] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (3) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 151.611 58.0556] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (H) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 174.389 59.8333] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (4) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 170.472 58.0556] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (H) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 136.722 59.6667] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (2) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 132.667 58.0556] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (H) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 114.194 96.8611] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (O) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 117.778 98.6111] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (1) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 132.75 96.8611] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (O) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 136.722 98.6944] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (2) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 151.611 96.8611] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (O) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 155.667 98.4722] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (3) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 170.472 96.8611] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (O) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 174.306 98.5556] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (4) s savemat setmatrix userdict /#copies 1 put showpage grestore end restore %%EndDocument endTexFig 0 818 a Fr(Figure)16 b(6:)j(\(a\))c(A)g(directed)h(graph)f(represen)o (tation)h(of)e(an)i(HMM.)e(Eac)o(h)h(horizon)o(tal)h(link)g(is)g(asso)q (ciated)g(with)0 874 y(the)h(transition)g(matrix)g Fm(A)p Fr(,)g(and)h(eac)o(h)f(v)o(ertical)g(link)i(is)e(asso)q(ciated)h(with)f (the)g(emission)h(matrix)f Fm(B)r Fr(.)26 b(\(b\))17 b(An)0 931 y(HMM)j(as)h(a)f(Boltzmann)h(mac)o(hine.)38 b(The)21 b(parameters)f(on)g(the)h(horizon)o(tal)g(links)h(are)f (logarithms)g(of)f(the)0 987 y(en)o(tries)e(of)f(the)g Fm(A)h Fr(matrix,)f(and)h(the)f(parameters)g(on)g(the)h(v)o(ertical)g (links)h(are)e(logarithms)g(of)g(the)h(en)o(tries)f(of)0 1044 y(the)e Fm(B)j Fr(matrix.)i(The)15 b(t)o(w)o(o)f(represen)o (tations)h(yield)i(the)e(same)g(join)o(t)g(probabilit)o(y)i (distribution.)0 1177 y(follo)o(wing)f(Mark)o(o)o(v)d(conditions:)406 1279 y Fm(H)444 1286 y Fi(i)471 1279 y Fk(?)g(f)p Fm(H)580 1286 y Fj(1)599 1279 y Fm(;)8 b(O)655 1286 y Fj(1)673 1279 y Fm(;)g(:)g(:)g(:)d(;)j(H)813 1286 y Fi(i)p Fl(\000)p Fj(2)871 1279 y Fm(;)g(O)927 1286 y Fi(i)p Fl(\000)p Fj(2)985 1279 y Fm(;)g(O)1041 1286 y Fi(i)p Fl(\000)p Fj(1)1099 1279 y Fk(gj)p Fm(H)1173 1286 y Fi(i)p Fl(\000)p Fj(1)1232 1279 y Fm(;)98 b Fr(2)12 b Fk(\024)h Fm(i)f Fk(\024)h Fm(N)330 b Fr(\(36\))0 1381 y(and)482 1438 y Fm(O)517 1445 y Fi(i)544 1438 y Fk(?)13 b(f)p Fm(H)653 1445 y Fj(1)672 1438 y Fm(;)8 b(O)728 1445 y Fj(1)746 1438 y Fm(;)g(:)g(:)g(:)d(;)j(H)886 1445 y Fi(i)p Fl(\000)p Fj(1)944 1438 y Fm(;)g(O)1000 1445 y Fi(i)p Fl(\000)p Fj(1)1058 1438 y Fk(gj)p Fm(H)1132 1445 y Fi(i)1146 1438 y Fm(;)98 b Fr(2)12 b Fk(\024)h Fm(i)f Fk(\024)h Fm(N)r(;)401 b Fr(\(37\))0 1521 y(where)15 b(the)h(sym)o(b)q(ol)f Fk(?)h Fr(is)f(used)h(to)f(denote)g(indep)q(endenc)q(e.)91 1578 y(Figure)d(6\(b\))f(sho)o(ws)h(that)f(it)h(is)h(p)q(ossible)g(to)f (treat)e(an)i(HMM)g(as)f(a)h(sp)q(ecial)i(case)e(of)f(a)h(Boltzmann)g (mac)o(hine)0 1634 y([Luttrell,)j(1989];)e([Saul)i(and)g(Jordan,)f (1995].)k(The)d(probabilistic)i(structure)d(of)g(the)h(HMM)f(can)g(b)q (e)i(captured)0 1691 y(b)o(y)e(de\014ning)h(the)f(w)o(eigh)o(ts)g(on)g (the)g(links)h(as)f(the)g(logarithms)g(of)f(the)i(corresp)q(onding)f (transition)h(and)f(emission)0 1747 y(probabilities.)40 b(The)22 b(Boltzmann)g(distribution)h(\(Eq.)d(35\))g(then)i(con)o(v)o (erts)f(the)g(additiv)o(e)h(energy)g(in)o(to)f(the)0 1804 y(pro)q(duct)c(form)f(of)g(the)h(standard)f(HMM)g(probabililit)o (y)j(distribution.)26 b(As)17 b(w)o(e)f(will)i(see,)f(this)g(reduction) h(of)e(a)0 1860 y(directed)g(graph)f(to)g(an)g(undirected)i(graph)e(is) g(a)g(recurring)h(theme)g(in)g(the)f(graphical)h(mo)q(del)g(formalism.) 91 1916 y(General)f(mixture)g(mo)q(dels)g(are)g(readily)h(view)o(ed)f (as)f(graphical)i(mo)q(dels)f([Bun)o(tine,)g(1994].)k(F)l(or)13 b(example,)0 1973 y(the)e(unconditional)h(mixture)f(mo)q(del)h(of)e (Eq.)g(2)g(can)h(b)q(e)g(represen)o(ted)g(as)f(a)g(graphical)i(mo)q (del)f(with)g(t)o(w)o(o)e(no)q(des|a)0 2029 y(m)o(ultinomial)15 b(\\hidden")f(no)q(de)g(whic)o(h)h(represen)o(ts)e(the)g(selected)i (comp)q(onen)o(t,)e(a)g(\\visible")i(no)q(de)f(represen)o(ting)0 2086 y Fn(x)p Fr(,)i(and)h(a)g(directed)g(link)i(from)d(the)g(hidden)j (no)q(de)e(to)f(the)h(visible)i(no)q(de)e(\(see)g(b)q(elo)o(w)g(for)f (the)h(hidden/visibl)q(e)0 2142 y(distinction\).)k(Conditional)c (mixture)e(mo)q(dels)h([Jacobs,)e(et)h(al.,)g(1991])e(simply)k(require) e(another)g(visible)i(no)q(de)0 2199 y(with)k(directed)g(links)h(to)d (the)i(hidden)h(no)q(de)f(and)f(the)h(visible)h(no)q(des.)36 b(Hierarc)o(hical)22 b(conditional)g(mixture)0 2255 y(mo)q(dels)17 b([Jordan)e(and)h(Jacobs,)g(1994])e(require)j(a)f(c)o(hain)g(of)g (hidden)h(no)q(des,)f(one)g(hidden)i(no)q(de)e(for)g(eac)o(h)g(lev)o (el)0 2312 y(of)f(the)g(tree.)91 2368 y(Within)g(the)e(general)h (framew)o(ork)f(of)g(probabilistic)j(graphical)e(mo)q(dels,)g(it)g(is)g (p)q(ossible)i(to)d(tac)o(kle)g(general)0 2425 y(problems)21 b(of)e(inference)j(and)e(learning.)35 b(The)20 b(k)o(ey)g(problem)g (that)g(arises)g(in)g(this)h(setting)f(is)g(the)g(problem)0 2481 y(of)c(computing)i(the)f(probabilities)i(of)d(certain)h(no)q(des,) h(whic)o(h)f(w)o(e)g(will)h(refer)f(to)f(as)h Fq(hidden)h(no)n(des)p Fr(,)e(giv)o(en)h(the)0 2537 y(observ)o(ed)12 b(v)m(alues)h(of)e(other) g(no)q(des,)i(whic)o(h)f(w)o(e)g(will)h(refer)f(to)f(as)g Fq(visible)h(no)n(des)p Fr(.)18 b(F)l(or)11 b(example,)i(in)f(an)g (HMM,)f(the)952 2775 y(22)p eop %%Page: 21 9 21 8 bop 525 150 a 14208860 8951580 4012687 27496775 31246336 44731596 startTexFig 525 150 a %%BeginDocument: ps/independence.ps save userdict /IslandDrawDict 300 dict dup begin put /ncpoint errordict /nocurrentpoint get def errordict begin /nocurrentpoint { dup /pathbbox load eq { pop 0 0 1 1 } { ncpoint } ifelse } bind def end /image_raster { %% sw sh dw dh xs ys translate scale /sh exch def /sw exch def /imagebuf sw 7 add 8 idiv string def sw sh 1 [sw 0 0 sh 0 0] { currentfile imagebuf readhexstring pop } image } bind def /m {moveto} bind def /l {lineto} bind def /c {curveto} bind def /n {newpath} bind def /cl {closepath} bind def /ar { %% sa ea sx sy rot tx ty matrix currentmatrix 8 1 roll translate rotate scale n 0 0 1 5 3 roll arc setmatrix } bind def /arn { %% sa ea sx sy rot tx ty matrix currentmatrix 8 1 roll translate rotate scale n 0 0 1 5 3 roll arcn setmatrix } bind def /el { %% sx sy rot tx ty matrix currentmatrix 6 1 roll translate rotate scale n 0 0 1 0 360 arc setmatrix cl } bind def /bp {setlinejoin setlinewidth setrgbcolor} bind def /bpbw {setlinejoin setlinewidth setgray} bind def /lw {setlinewidth} bind def /lj {setlinejoin} bind def /gr {setgray} bind def /BPSIDE 32 def %% pixels per pattern side /PATFREQ 3.0 def %% pattern pixels per mm /dp_mat [PATFREQ 0 0 PATFREQ 0 0] def /dp_pw BPSIDE def %% pattern pixel width /dp_ph BPSIDE def %% pattern pixel height /dp_w dp_pw PATFREQ div def %% pattern mm width /dp_h dp_ph PATFREQ div def %% pattern mm height /dp_bs 1 def %% pattern bits per pixel /savemat matrix def /topmat matrix def /patmat matrix def /patpath { /inv exch def topmat setmatrix pathbbox %% get lo - hi indecies /hy exch dp_h div floor cvi def /hx exch dp_w div floor cvi def /ly exch dp_h div floor cvi def /lx exch dp_w div floor cvi def lx 1 hx { dp_w mul ly 1 hy { dp_h mul exch dup 3 1 roll exch patmat currentmatrix pop translate dp_pw dp_ph inv dp_mat dp_proc imagemask patmat setmatrix } for pop } for } bind def /setpattern { /blue exch def /green exch def /red exch def /freq exch def /bwidth exch def /bpside exch def /bstring exch def /onbits 0 def /offbits 0 def freq 0 {/y exch def /x exch def /xindex x 1 add 2 div bpside mul cvi def /yindex y 1 add 2 div bpside mul cvi def bstring yindex bwidth mul xindex 8 idiv add get not 1 7 xindex 8 mod sub bitshift and 0 ne {/onbits onbits 1 add def 1} {/offbits offbits 1 add def 0} ifelse } setscreen {} settransfer systemdict /setcmykcolor known { /fact 1 onbits offbits onbits add div sub def 1 red sub fact mul 1 green sub fact mul 1 blue sub fact mul 0 setcmykcolor } { offbits offbits onbits add div setgray } ifelse } bind def /dmatrix matrix def /dpi 72 0 dmatrix defaultmatrix dtransform dup mul exch dup mul add sqrt def /B {gsave bp stroke grestore} bind def %% brush: gr lw lj /Bbw {gsave bpbw stroke grestore} bind def %% brush: gr lw lj /F {gsave setrgbcolor eofill grestore} bind def %% fill: gr /Fbw {gsave setgray eofill grestore} bind def %% fill: gr /PB {gsave setlinejoin setlinewidth setpattern stroke grestore} bind def /PF {gsave eoclip patpath grestore} bind def /BB { gsave setrgbcolor setlinejoin setlinewidth strokepath clip patpath grestore } bind def /BLACK { 0.0 } bind def /CP {closepath} bind def /FI {eofill} bind def /E {exch} bind def /FF {findfont} bind def /GR {grestore} bind def /GS {gsave} bind def /MF {makefont} bind def /NP {newpath} bind def /RO {rotate} bind def /ST {stroke} bind def /SC {scale} bind def /SF {setfont} bind def /SG {setgray} bind def /SLC {setlinecap} bind def /SLJ {setlinejoin} bind def /SLW {setlinewidth} bind def /TR {translate} bind def /WHITE { 1.0 } bind def /m {moveto} bind def /r {rmoveto} bind def /l {lineto} bind def /sp {x 0 rmoveto} bind def /rl {rlineto} bind def /s {show} bind def /box { NP m l l l CP } bind def /pageboundary { NP m l l l CP } bind def /BS { % black stroke GS SLJ SLW BLACK SG ST GR } bind def /WS { % white stroke GS SLJ SLW WHITE SG ST GR } bind def /reencode_small_dict 12 dict def /ReencodeSmall { reencode_small_dict begin /new_codes_and_names E def /new_font_name E def /base_font_name E def /base_font_dict base_font_name FF def /newfont base_font_dict maxlength dict def base_font_dict { E dup /FID ne { dup /Encoding eq { E dup length array copy newfont 3 1 roll put } { E newfont 3 1 roll put } ifelse } { pop pop } ifelse } forall newfont /FontName new_font_name put new_codes_and_names aload pop new_codes_and_names length 2 idiv { newfont /Encoding get 3 1 roll put } repeat new_font_name newfont definefont pop end %reencode_small_dict } def /extended_Zapf [ 8#223 /a89 8#224 /a90 8#225 /a93 8#226 /a94 8#227 /a91 8#230 /a92 8#231 /a205 8#232 /a85 8#233 /a206 8#234 /a86 8#235 /a87 8#236 /a88 8#237 /a95 8#240 /a96 ] def /extended_Standard [ 29 /thorn 30 /yacute 31 /divide 128 /Acircumflex 129 /Adieresis 130 /Agrave 131 /Aring 132 /Atilde 133 /Ccedilla 134 /Eacute 135 /Ecircumflex 136 /Edieresis 137 /Egrave 138 /Iacute 139 /Icircumflex 140 /Idieresis 141 /Igrave 142 /Ntilde 143 /Oacute 144 /Ocircumflex 145 /Odieresis 146 /Ograve 147 /Otilde 148 /Scaron 149 /Uacute 150 /Ucircumflex 151 /Udieresis 152 /Ugrave 153 /Ydieresis 154 /Zcaron 155 /aacute 156 /acircumflex 157 /adieresis 158 /agrave 159 /aring 160 /atilde 161 /exclamdown 162 /cent 163 /sterling 164 /fraction 165 /yen 166 /florin 167 /section 168 /currency 169 /quotesingle 170 /quotedblleft 171 /guillemotleft 172 /guilsinglleft 173 /guilsinglright 174 /fi 175 /fl 176 /plusminus 177 /endash 178 /dagger 179 /daggerdbl 180 /periodcentered 181 /twosuperior 182 /paragraph 183 /bullet 184 /quotesinglebase 185 /quotedblbase 186 /quotedblright 187 /guillemotright 188 /ellipsis 189 /perthousand 190 /threesuperior 191 /questiondown 192 /mu 193 /grave 194 /acute 195 /circumflex 196 /tilde 197 /macron 198 /breve 199 /dotaccent 200 /dieresis 201 /onesuperior 202 /ring 203 /cedilla 204 /onequarter 205 /hungarumlaut 206 /ogonek 207 /caron 208 /emdash 209 /ccedilla 210 /copyright 211 /eacute 212 /ecircumflex 213 /edieresis 214 /egrave 215 /iacute 216 /icircumflex 217 /idieresis 218 /igrave 219 /logicalnot 220 /minus 221 /ntilde 222 /oacute 223 /ocircumflex 224 /odieresis 225 /AE 226 /onehalf 227 /ordfeminine 228 /ograve 229 /otilde 230 /registered 231 /scaron 232 /Lslash 233 /Oslash 234 /OE 235 /ordmasculine 236 /trademark 237 /uacute 238 /ucircumflex 239 /udieresis 240 /ugrave 241 /ae 242 /ydieresis 243 /zcaron 244 /Aacute 245 /dotlessi 246 /threequarters 247 /Eth 248 /lslash 249 /oslash 250 /oe 251 /germandbls 252 /multiply 253 /Yacute 254 /Thorn 255 /eth ] def /extended_Symbol [ ] def /extend_font { % stack: fontname newfontname E dup (ZapfDingbats) eq { cvn E cvn extended_Zapf ReencodeSmall } { dup (Symbol) eq { cvn E cvn extended_Symbol ReencodeSmall } { cvn E cvn extended_Standard ReencodeSmall } ifelse } ifelse } bind def /getfont { /f E def f cvn where { begin f cvn load exec SF end } { f 0 f length 8 sub getinterval dup dup length 1 add string /localfont exch def localfont exch 0 exch putinterval localfont dup length 1 sub (X) putinterval localfont extend_font localfont FF /xsz f f length 4 sub 4 getinterval cvi def /ysz f f length 8 sub 4 getinterval cvi def [ xsz 0 0 ysz neg 0 0 ] MF dup f cvn E def SF } ifelse } bind def /ul { % space drop thickness GS currentpoint currentlinewidth currentpoint NP m 6 -3 roll SLW 0 E r 0 rl ST SLW m GR } bind def /ss { currentpoint pop E m } bind def /image_raster { % sw sh dw dh xs ys TR SC /sh E def /sw E def /imagebuf sw 7 add 8 idiv string def sw sh 1 [sw 0 0 sh 0 0] { currentfile imagebuf readhexstring pop } image } bind def /imagemask_raster { TR SC /sh E def /sw E def /imagebuf sw 7 add 8 idiv string def sw sh false [sw 0 0 sh 0 0] { currentfile imagebuf readhexstring pop } imagemask } bind def /image_color_raster { % sw sh sd dw dh xs ys systemdict /colorimage known not { /colorimage /colimg load def } if TR SC /sd E def /sh E def /sw E def /imagebuf sw 3 mul sd mul 7 add 8 idiv string def sw sh sd [sw 0 0 sh 0 0] { currentfile imagebuf readhexstring pop } false 3 colorimage } bind def /nx { /x E def } bind def 0. nx gsave 2.83465 -2.83465 scale 0 -279.4 translate topmat currentmatrix pop 5.0855 5.0855 0 143.42 94.222 el 1 Fbw gsave 0 0.3528 0 Bbw grestore n 36.361 77.611 m 58.972 55 l 1 Fbw gsave 0 0.3528 0 Bbw grestore n 59.5 99.528 m 82.111 76.917 l 1 Fbw gsave 0 0.3528 0 Bbw grestore n 81.417 77.75 m 58.806 55.139 l 1 Fbw gsave 0 0.3528 0 Bbw grestore n 59.333 99.972 m 36.722 77.361 l 1 Fbw gsave 0 0.3528 0 Bbw grestore 5.0855 5.0855 0 59.278 55 el 1 Fbw gsave 0 0.3528 0 Bbw grestore 5.0855 5.0855 0 36.583 77.528 el 1 Fbw gsave 0 0.3528 0 Bbw grestore 5.0855 5.0855 0 82.333 77.75 el 1 Fbw gsave 0 0.3528 0 Bbw grestore 5.0855 5.0855 0 59.333 99.972 el 1 Fbw gsave 0 0.3528 0 Bbw grestore n 125.58 67.833 m 140.56 89.833 l 1 Fbw n 140.56 89.833 m 137.57 87.559 l 139.53 86.224 l cl 0 Fbw n 125.58 67.833 m 138.55 86.892 l gsave 0 0.3528 0 Bbw grestore n 161.25 67.75 m 146.28 89.75 l 1 Fbw n 146.28 89.75 m 147.3 86.141 l 149.26 87.476 l cl 0 Fbw n 161.25 67.75 m 148.28 86.808 l gsave 0 0.3528 0 Bbw grestore 5.0855 5.0855 0 125.25 67.194 el 1 Fbw gsave 0 0.3528 0 Bbw grestore 5.0855 5.0855 0 161.22 67.417 el 1 Fbw gsave 0 0.3528 0 Bbw grestore n savemat currentmatrix pop [1 0 0 1 21.6944 79.9722] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic04000400) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 25.6667 81.5] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (i) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 91.5833 79.9722] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic04000400) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 95.6389 80.9722] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (j) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 56.9722 45.3611] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic04000400) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 61.1111 47.3611] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (k) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 56.9722 114.028] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic04000400) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 60.8889 116.111] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (l) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 120.306 57.8056] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic04000400) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 124.278 59.3333] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (i) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 141.306 109.222] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic04000400) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 145.361 110.222] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (j) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 158.333 57.8056] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic04000400) getfont () s 0.00 SG (X) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 162.472 59.8056] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (k) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 55.6111 130.778] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Helvetica03600360) getfont () s 0.00 SG (\(a\)) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 139.944 130.778] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Helvetica03600360) getfont () s 0.00 SG (\(b\)) s savemat setmatrix userdict /#copies 1 put showpage grestore end restore %%EndDocument endTexFig 0 815 a Fr(Figure)19 b(5:)26 b(\(a\))18 b(An)h(undirected)h(graph)f (in)g(whic)o(h)h Fm(X)953 822 y Fi(i)985 815 y Fr(is)f(indep)q(enden)o (t)i(of)d Fm(X)1388 822 y Fi(j)1425 815 y Fr(giv)o(en)h Fm(X)1585 822 y Fi(k)1625 815 y Fr(and)f Fm(X)1754 822 y Fi(l)1767 815 y Fr(,)h(and)g Fm(X)1929 822 y Fi(k)0 871 y Fr(is)f(indep)q(enden)o(t)h(of)e Fm(X)399 878 y Fi(l)429 871 y Fr(giv)o(en)g Fm(X)587 878 y Fi(i)618 871 y Fr(and)g Fm(X)746 878 y Fi(j)764 871 y Fr(.)26 b(\(b\))17 b(A)g(directed)h(graph)f(in)h(whic)o(h)g Fm(X)1464 878 y Fi(i)1495 871 y Fr(and)f Fm(X)1623 878 y Fi(k)1661 871 y Fr(are)g(marginally)0 928 y(indep)q(enden)o(t)h(but)d(are)g (conditionally)i(dep)q(enden)o(t)g(giv)o(en)f Fm(X)1065 935 y Fi(j)1083 928 y Fr(.)0 1066 y(indep)q(endence)i(in)e(directed)g (graphs)e(is)h(captured)h(b)o(y)e(a)h(graphical)g(criterion)h(kno)o(wn) f(as)f Fq(d-sep)n(ar)n(ation)h Fr([P)o(earl,)0 1122 y(1988],)e(whic)o (h)i(di\013ers)f(from)g(undirected)i(separation)e(only)h(in)g(those)f (cases)g(in)h(whic)o(h)h(paths)e(ha)o(v)o(e)f(t)o(w)o(o)g(arro)o(ws)0 1179 y(arriving)j(at)e(the)i(same)e(no)q(de)i(\(as)f(in)h(Figure)f (5\(b\)\).)91 1235 y(Although)21 b(the)f(neural)i(net)o(w)o(ork)d(arc)o (hitectures)i(that)f(w)o(e)g(ha)o(v)o(e)g(discussed)i(un)o(til)f(no)o (w)f(ha)o(v)o(e)g(all)i(b)q(een)0 1292 y(based)f(on)g(directed)h (graphs,)f(undirected)i(graphs)d(also)h(pla)o(y)g(an)g(imp)q(ortan)o(t) g(role)g(in)g(neural)h(net)o(w)o(ork)e(re-)0 1348 y(searc)o(h.)k (Constrain)o(t)16 b(satisfaction)h(arc)o(hitectures,)g(including)j(the) d(Hop\014eld)h(net)o(w)o(ork)e([Hop\014eld,)h(1982])f(and)0 1404 y(the)e(Boltzmann)h(mac)o(hine)g([Hin)o(ton)g(and)f(Sejno)o(wski,) h(1986],)d(are)i(the)h(most)e(prominen)o(t)i(examples.)20 b(A)14 b(Boltz-)0 1461 y(mann)21 b(mac)o(hine)i(is)e(an)h(undirected)h (probabilistic)g(graph)e(that)g(resp)q(ects)g(the)h(conditional)h (indep)q(endency)0 1517 y(seman)o(tics)13 b(describ)q(ed)j(ab)q(o)o(v)o (e)d(\(cf.)19 b(Figure)13 b(5\(a\)\).)18 b(Eac)o(h)13 b(no)q(de)h(in)h(a)e(Boltzmann)g(mac)o(hine)i(is)f(a)f(binary-v)m (alued)0 1574 y(random)h(v)m(ariable)i Fm(X)374 1581 y Fi(i)403 1574 y Fr(\(or)d(more)i(generally)g(a)g(discrete-v)m(alued)i (random)d(v)m(ariable\).)21 b(A)15 b(probabilit)o(y)g(distribu-)0 1630 y(tion)j(on)f(the)h(2)265 1614 y Fi(N)315 1630 y Fr(p)q(ossible)i(con\014gurations)d(of)g(suc)o(h)h(v)m(ariables)h(is)f (de\014ned)g(via)g(an)f Fq(ener)n(gy)h(function)f Fm(E)s Fr(.)26 b(Let)0 1687 y Fm(J)25 1694 y Fi(ij)70 1687 y Fr(b)q(e)16 b(the)f(w)o(eigh)o(t)g(on)f(the)h(link)i(b)q(et)o(w)o(een)e Fm(X)797 1694 y Fi(i)825 1687 y Fr(and)h Fm(X)952 1694 y Fi(j)969 1687 y Fr(,)f(let)g Fm(J)1087 1694 y Fi(ij)1130 1687 y Fr(=)e Fm(J)1203 1694 y Fi(j)r(i)1234 1687 y Fr(,)h(let)i Fm(\013)f Fr(index)h(the)f(con\014gurations,)g(and)0 1743 y(de\014ne)h(the)g(energy)f(of)g(con\014guration)g Fm(\013)g Fr(as)g(follo)o(ws:)760 1845 y Fm(E)794 1852 y Fi(\013)831 1845 y Fr(=)e Fk(\000)922 1805 y Ff(X)924 1896 y Fi(i<j)990 0="" 1="" 2="" 3="" 4="" 5="" 6="" 7="" 8="" 9="" 10="" 11="" 12="" 13="" 14="" 15="" 16="" 17="" 18="" 19="" 20="" 21="" 23="" 24="" 25="" 26="" 27="" 29="" 30="" 31="" 32="" 36="" 40="" 50="" 66="" 67="" 69="" 73="" 76="" 82="" 87="" 91="" 93="" 96="" 100="" 132="" 150="" 175="" 192="" 199="" 202="" 214="" 216="" 219="" 243="" 248="" 255="" 266="" 271="" 277="" 304="" 310="" 331="" 334="" 350="" 360="" 361="" 388="" 390="" 401="" 417="" 442="" 447="" 468="" 474="" 489="" 503="" 530="" 531="" 542="" 543="" 545="" 560="" 561="" 563="" 567="" 571="" 587="" 589="" 596="" 602="" 616="" 623="" 625="" 626="" 643="" 652="" 658="" 673="" 680="" 681="" 700="" 706="" 708="" 710="" 715="" 717="" 729="" 738="" 748="" 755="" 756="" 759="" 762="" 769="" 771="" 774="" 779="" 785="" 792="" 794="" 795="" 813="" 819="" 826="" 828="" 833="" 834="" 842="" 849="" 850="" 851="" 854="" 857="" 869="" 876="" 884="" 890="" 898="" 907="" 914="" 916="" 925="" 926="" 934="" 937="" 940="" 941="" 944="" 951="" 952="" 963="" 964="" 970="" 974="" 979="" 982="" 986="" 988="" 989="" 990="" 997="" 1008="" 1010="" 1012="" 1019="" 1020="" 1021="" 1025="" 1026="" 1027="" 1028="" 1029="" 1032="" 1038="" 1039="" 1045="" 1052="" 1053="" 1064="" 1076="" 1077="" 1083="" 1095="" 1097="" 1110="" 1113="" 1114="" 1119="" 1120="" 1122="" 1133="" 1140="" 1144="" 1145="" 1151="" 1152="" 1160="" 1166="" 1170="" 1173="" 1176="" 1180="" 1187="" 1189="" 1196="" 1201="" 1208="" 1214="" 1223="" 1227="" 1233="" 1240="" 1246="" 1252="" 1258="" 1259="" 1264="" 1269="" 1279="" 1283="" 1288="" 1289="" 1295="" 1300="" 1302="" 1305="" 1308="" 1318="" 1321="" 1322="" 1329="" 1330="" 1335="" 1336="" 1338="" 1339="" 1346="" 1348="" 1350="" 1353="" 1355="" 1359="" 1368="" 1376="" 1377="" 1379="" 1386="" 1392="" 1393="" 1396="" 1402="" 1410="" 1415="" 1422="" 1429="" 1434="" 1439="" 1446="" 1449="" 1452="" 1456="" 1458="" 1469="" 1472="" 1490="" 1496="" 1509="" 1515="" 1517="" 1526="" 1528="" 1546="" 1552="" 1565="" 1570="" 1571="" 1574="" 1582="" 1585="" 1603="" 1609="" 1612="" 1622="" 1628="" 1630="" 1641="" 1656="" 1659="" 1665="" 1668="" 1678="" 1684="" 1687="" 1698="" 1713="" 1716="" 1722="" 1735="" 1741="" 1743="" 1754="" 1769="" 1772="" 1789="" 1791="" 1797="" 1800="" 1810="" 1825="" 1826="" 1829="" 1843="" 1845="" 1848="" 1852="" 1854="" 1856="" 1867="" 1882="" 1885="" 1889="" 1898="" 1902="" 1904="" 1910="" 1912="" 1923="" 1929="" 1938="" 1942="" 1954="" 1958="" 1960="" 1980="" 1985="" 1988="" 1995="" 1998="" 2002="" 2004="" 2005="" 2007="" 2009="" 2011="" 2015="" 2017="" 2022="" 2025="" 2032="" 2036="" 2039="" 2042="" 2043="" 2047="" 2051="" 2055="" 2061="" 2063="" 2067="" 2071="" 2072="" 2073="" 2076="" 2078="" 2083="" 2087="" 2088="" 2093="" 2098="" 2108="" 2111="" 2119="" 2120="" 2126="" 2128="" 2130="" 2137="" 2138="" 2142="" 2149="" 2152="" 2155="" 2163="" 2164="" 2167="" 2168="" 2175="" 2184="" 2186="" 2193="" 2206="" 2211="" 2218="" 2221="" 2224="" 2240="" 2243="" 2244="" 2245="" 2251="" 2253="" 2254="" 2256="" 2261="" 2262="" 2268="" 2272="" 2275="" 2277="" 2279="" 2280="" 2283="" 2285="" 2288="" 2295="" 2297="" 2299="" 2305="" 2308="" 2312="" 2319="" 2321="" 2322="" 2324="" 2327="" 2329="" 2334="" 2337="" 2340="" 2347="" 2351="" 2352="" 2353="" 2354="" 2356="" 2364="" 2371="" 2375="" 2381="" 2402="" 2410="" 2412="" 2415="" 2421="" 2428="" 2429="" 2431="" 2437="" 2444="" 2449="" 2460="" 2466="" 2468="" 2469="" 2476="" 2477="" 2488="" 2491="" 2494="" 2523="" 2525="" 2532="" 2534="" 2537="" 2544="" 2550="" 2557="" 2668="" 2696="" 2698="" 2716="" 2737="" 2763="" 2772="" 2775="" 2778="" 2802="" 2816="" 2940="" 2994="" 3067="" 3074="" 3230="" 3309="" 3427="" 3577="" 3625="" 3679="" 3705="" 3791="" 3862="" 3863="" 4086="" 4130="" 4281="" 4314="" 4334="" 4346="" 4348="" 4355="" 4381="" 4396="" 4448="" 4453="" 4558="" 4570="" 4612="" 4678="" 4692="" 4848="" 5184="" 5193="" 5359="" 5894="" 6106="" 6202="" 6256="" 6914="" 7344="" 4210032="" 13353697="" 17677154="" 22376157="" 36508876="" 39008583="" y="" fm(j)1015="" fi(ij)1045="" fm(x)1087="" fi(\013)1083="" y(i)1111="" fm(x)1153="" fi(\013)1149="" y(j)1177="" fm(:)679="" b="" fr(\(34\))0="" y(the)15="" b(probabilit)o(y)i(of)e="" (con\014guration)g="" fm(\013)g="" fr(is)h(then)g(de\014ned)g(via)g(the)f="" (boltzmann)h(distribution:)796="" fm(p)825="" fi(\013)863="" fr(=")951" fm(e)972="" fl(\000)p="" fi(e)1025="" fd(\013)1048="" fi(="T)p" v="" a="" ff(p)959="" fi(\015)989="" fm(e)1010="" fi(e)1063="" fd(\015)1084="" fm(;)722="" fr(\(35\))0="" y(where)15="" b(the)h="" fq(temp)n(er)n(atur)n(e)f="" fm(t)21="" fr(pro)o(vides)16="" b(a)f(scale)h(for)e(the)h(energy)l(.)91="" y(an)h(example)g(of)g(a)f(directed)i(probabilistic)h(graph)d(is)i="" (the)e(hidden)j(mark)o(o)o(v)c(mo)q(del)j(\(hmm\).)d(an)i(hmm)0="" y(is)e(de\014ned)g(b)o(y)f(a)g(set)g(of)g="" fq(state)h(variables)j="" fm(h)749="" fi(i)762="" fr(,)d(where)f="" fm(i)g="" fr(is)h(generally)g(a)f(time)g(or)g(a)g(space)g(index,)i(a)d(set)h(of)g="" (output)0="" y(v)m(ariables)g="" fm(o)220="" fi(i)234="" fr(,)e(a)h="" fq(pr)n(ob)n(ability)g(tr)n(ansition)g(matrix)18="" fm(a)13="" fm(p)p="" fr(\()p="" fm(h)1033="" fi(i)1046="" fk(j)p="" fm(h)1097="" fi(i)p="" fj(1)1156="" fr(\),)e(and)h(an)f="" fq(emission)h(matrix)h="" fm(b)i="" fm(o)1842="" fi(i)1855="" fm(h)1906="" fi(i)1920="" fr(\).)0="" y(the)19="" b(directed)i(graph)e(for)f(an)h(hmm)g(is)g(sho)o(wn)g(in)h="" (figure)f(6\(a\).)31="" b(as)19="" b(can)g(b)q(e)h(seen)g(from)e(considering)="" j(the)0="" y(separatory)14="" b(prop)q(erties)h(of)f(the)h(graph,)f(the)="" h(conditional)i(indep)q(endencie)q(s)g(of)e(the)f(hmm)h(are)f="" (de\014ned)i(b)o(y)f(the)952="" y(21)p="" eop="" %%page:="" bop="" fr(net)o(w)o(ork)10="" b(to)g(map)g(from)g(the)h(input)g="" (image)g(to)f(a)h(set)f(of)g(10)g(output)h(v)m(alues)h(represen)o(ting)="" f(p)q(osterior)g(probabilities)0="" y(for)j(the)h(10)g(classes.)20="" b(ho)o(w)o(ev)o(er,)14="" b(w)o(e)g(kno)o(w)h(that)f(the)h="" (classi\014cation)h(of)f(a)g(digit)g(should)h(b)q(e)g(indep)q(enden)o="" (t)h(of)e(its)0="" y(p)q(osition)i(within)g(the)f(input)h(image.)23="" b(one)16="" b(w)o(a)o(y)f(of)h(ac)o(hieving)h(suc)o(h)f="" fq(tr)n(anslation)g(invarianc)n(e)f="" fr(is)i(to)e(mak)o(e)h(use)0="" y(of)f(the)g(tec)o(hnique)h(of)e="" fq(shar)n(e)n(d)i(weights)p="" fr(.)k(this)15="" b(in)o(v)o(olv)o(es)h(a)e(net)o(w)o(ork)g(arc)o="" (hitecure)i(ha)o(ving)f(man)o(y)g(hidden)h(la)o(y)o(ers)0="" y(in)k(whic)o(h)g(eac)o(h)f(unit)g(tak)o(es)f(inputs)i(only)g(from)="" e(a)g(small)i(patc)o(h,)f(called)i(a)e="" fq(r)n(e)n(c)n(eptive)f(\014eld)="" p="" fr(,)h(of)f(units)i(in)g(the)0="" y(previous)d(la)o(y)o(er.)k(by)16="" b(a)f(pro)q(cess)h(of)g(constraining)g(neigh)o(b)q(oring)i(units)e(to)f="" (ha)o(v)o(e)h(common)f(w)o(eigh)o(ts,)g(it)h(can)g(b)q(e)0="" y(arranged)d(that)g(the)g(output)h(of)f(the)g(net)o(w)o(ork)g(is)h="" (insensitiv)o(e)i(to)c(translations)i(of)f(the)h(input)g(image.)20="" b(a)13="" b(further)0="" y(b)q(ene\014t)18="" b(of)e(w)o(eigh)o(t)g(sharing)="" g(is)h(that)f(the)h(n)o(um)o(b)q(er)f(of)g(indep)q(enden)o(t)j="" (parameters)d(is)h(m)o(uc)o(h)f(smaller)i(than)e(the)0="" y(n)o(um)o(b)q(er)e(of)f(w)o(eigh)o(ts,)g(whic)o(h)i(assists)e="" (with)h(the)g(problem)g(of)f(mo)q(del)i(complexit)o(y)l(.)20="" b(this)14="" b(approac)o(h)g(is)g(the)g(basis)0="" y(for)f(the)h(highly)i="" (successful)f(us)f(p)q(ostal)g(co)q(de)g(recognition)h(system)f(of)f="" ([lecun,)h(et)g(al.,)g(1989].)k(an)c(alternativ)o(e)0="" y(to)f(shared)h(w)o(eigh)o(ts)g(is)g(to)g(enlargen)g(the)g="" (training)h(set)e(arti\014cially)j(b)o(y)e(generating)g(\\virtual)g="" (examples")h(based)0="" y(on)f(applying)i(translations)f(and)g(other)f="" (transformations)f(to)h(the)g(original)i(training)f(set)f([p)o(oggio)g="" (and)h(v)l(etter,)0="" y(1992].)0="" fp(4)69="" b(graphical)22="" b(mo)r(dels)0="" fr(neural)16="" b(net)o(w)o(orks)d(express)j="" (relationships)g(b)q(et)o(w)o(een)g(v)m(ariables)g(b)o(y)f(utilizing)i="" (the)e(represen)o(tational)h(language)0="" y(of)h(graph)g(theory)l(.)="" b(v)l(ariables)18="" b(are)f(asso)q(ciated)g(with)h(no)q(des)f(in)h(a)f="" (graph)g(and)g(transformations)f(of)h(v)m(ariables)0="" y(are)h(based)g(on)g(algorithms)g(that)g(propagate)f(n)o(umerical)="" i(messages)f(along)g(the)g(links)h(of)f(the)g(graph.)28="" b(more-)0="" y(o)o(v)o(er,)16="" b(the)h(graphs)g(are)f(often)h="" (accompanied)h(b)o(y)f(probabilistic)i(in)o(terpretations)e(of)f(the)h="" (v)m(ariables)h(and)f(their)0="" y(in)o(terrelationships.)k(as)14="" b(w)o(e)g(ha)o(v)o(e)f(seen,)i(suc)o(h)f(probabilistic)i(in)o="" (terpretations)e(allo)o(w)g(a)g(neural)g(net)o(w)o(ork)f(to)g(b)q(e)0="" y(understo)q(o)q(d)k(as)f(a)h(form)e(of)i(probabilistic)i(mo)q="" (del,)e(and)g(reduce)g(the)g(problem)g(of)f(learning)i(the)f(w)o(eigh)o="" (ts)f(of)g(a)0="" y(net)o(w)o(ork)e(to)h(a)g(problem)g(in)h="" (statistics.)91="" y(related)d(graphical)g(mo)q(dels)g(ha)o(v)o(e)f="" (b)q(een)h(studied)g(throughout)e(statistics,)h(engineering)i(and)f(ai)="" f(in)h(recen)o(t)0="" y(y)o(ears.)19="" b(hidden)c(mark)o(o)o(v)d(mo)q="" (dels,)j(kalman)f(\014lters,)g(and)g(path)g(analysis)g(mo)q(dels)h(are)="" e(all)i(examples)g(of)e(graph-)0="" y(ical)k(probabilistic)h(mo)q="" (dels)f(that)e(can)h(b)q(e)g(\014tted)g(to)f(data)g(and)h(used)h(to)e="" (mak)o(e)g(inferences.)23="" b(the)16="" b(relationship)0="" y(b)q(et)o(w)o(een)j(these)f(mo)q(dels)h(and)f(neural)i(net)o(w)o(orks)="" d(is)h(rather)g(strong;)g(indeed)i(it)f(is)g(often)f(p)q(ossible)i(to)d="" (reduce)0="" y(one)f(kind)g(of)f(mo)q(del)h(to)f(the)g(other.)21="" b(in)16="" b(this)f(section,)h(w)o(e)f(examine)h(these)g(relationships)h="" (in)f(some)f(detail)i(and)0="" y(pro)o(vide)f(a)f(broader)g(c)o="" (haracterization)h(of)e(neural)j(net)o(w)o(orks)d(as)h(mem)o(b)q(ers)g="" (of)g(a)g(general)h(family)g(of)f(graphical)0="" y(probabilistic)i="" (mo)q(dels.)91="" y(man)o(y)d(in)o(teresting)i(relationships)h(ha)o="" (v)o(e)d(b)q(een)i(disco)o(v)o(ered)g(b)q(et)o(w)o(een)f(graphs)g(and)g="" (probabilit)o(y)h(distribu-)0="" y(tions)h([spiegelhalter,)h(et)f="" (al.,)f(1993];)f([p)o(earl,)i(1988].)22="" b(these)17="" b(relationships)h="" (deriv)o(e)g(from)e(the)g(use)h(of)f(graphs)0="" y(to)i(represen)o(t)="" g(conditional)j(indep)q(endencie)q(s)g(among)d(random)g(v)m(ariables.)="" b(in)19="" b(an)g(undirected)h(graph,)f(there)0="" y(is)d(a)f(direct)i(corresp)q(ondence)f(b)q(et)o(w)o(een)g(conditional)="" i(indep)q(endence)h(and)c(graph)h(separation|random)f(v)m(ari-)0="" y(ables)21="" fm(x)157="" fi(i)191="" fr(and)g="" fm(x)323="" fi(k)364="" fr(are)g(conditionally)h(indep)q(enden)="" o(t)h(giv)o(en)e="" fm(x)1147="" fi(j)1185="" fr(if)g(no)q(des)g="" fm(x)1403="" fi(i)1438="" fr(and)f="" fm(x)1569="" fi(k)1611="" fr(are)g(separated)g(b)o(y)0="" y(no)q(de)f="" fm(x)151="" fi(j)187="" fr(\(w)o(e)f(use)g="" (the)g(sym)o(b)q(ol)h(\\)p="" fm(x)657="" fi(i)670="" fr(")f(to)g(represen)o(t)g(b)q(oth)h(a)e(random)h(v)m(ariable)i(and)e="" (a)g(no)q(de)h(in)g(a)f(graph\).)0="" y(this)d(statemen)o(t)f="" (remains)h(true)g(for)f(sets)h(of)f(no)q(des)i(\(see)f(figure)g="" (5\(a\)\).)j(directed)d(graphs)g(ha)o(v)o(e)f(a)h(somewhat)0="" y(di\013eren)o(t)g(seman)o(tics,)g(due)h(to)f(the)g(abilit)o(y)h="" (of)f(directed)i(graphs)d(to)h(represen)o(t)g(\\induced)i(dep)q="" (endencies.")23="" b(an)0="" y(induced)c(dep)q(endency)h(is)d(a)g="" (situation)h(in)g(whic)o(h)g(t)o(w)o(o)e(no)q(des)i(whic)o(h)g(are)f="" (marginally)h(indep)q(enden)o(t)h(b)q(ecome)0="" y(conditionally)g="" (dep)q(enden)o(t)f(giv)o(en)g(the)f(v)m(alue)h(of)e(a)h(third)g(no)q="" (de)h(\(see)e(figure)i(5\(b\)\).)23="" b(supp)q(ose,)18="" b(for)e(example,)0="" y(that)d="" fm(x)135="" fi(i)163="" fr(and)h="" fm(x)288="" fi(k)323="" fr(represen)o(t)g="" (indep)q(enden)o(t)i(coin)f(tosses,)e(and)h="" fm(x)1135="" fi(j)1167="" fr(represen)o(ts)g(the)g(sum)g(of)f="" fm(x)1640="" fi(i)1668="" fm(x)1793="" fi(k)1814="" fr(.)19="" b(then)0="" fm(x)38="" fi(i)64="" fr(and)13="" fm(x)188="" fi(k)222="" fr(are)f(marginally)i(indep)q(enden)o(t)h(but)e(are)f="" (conditionally)j(dep)q(enden)o(t)g(giv)o(en)e="" fi(j)1587="" b(the)13="" b(seman)o(tics)f(of)952="" y(20)p="" fr(to)16="" b(linearize)j="" fn(t)p="" fn(x)p="" fm(;)8="" fn(w)q="" fr(\))15="" b(ab)q(out)i="" fn(w)603="" fj(mp)678="" fr(so)f(that)h(the)g(in)o(tegration)g="" (can)g(b)q(e)g(p)q(erformed)g(analytically)i([macka)o(y)l(,)0="" y(1992].)f(alternativ)o(ely)l(,)d(sophisticated)f(mon)o(te)f(carlo)="" g(metho)q(ds)h(can)g(b)q(e)g(emplo)o(y)o(ed)g(to)f(ev)m(aluate)h(the)g="" (in)o(tegrals)0="" y(n)o(umerically)19="" b([neal,)e(1994].)24="" b(an)18="" b(imp)q(ortan)o(t)e(asp)q(ect)i(of)e(the)h(ba)o(y)o(esian)h="" (approac)o(h)e(is)i(that)e(there)h(is)h(no)f(need)0="" y(to)h(k)o(eep)h(data)e(aside)i(in)h(a)e(v)m(alidation)i(set)e(as)g(is)="" h(required)h(when)f(using)g(maxim)o(um)f(lik)o(eliho)q(o)q(d.)33="" b(practi-)0="" y(cal)g(applications)h(for)e(whic)o(h)h(the)f="" (quan)o(tit)o(y)g(of)g(a)o(v)m(ailable)i(data)e(are)g(limited,)j(it)e="" (is)f(found)h(that)f(a)g(ba)o(y)o(esian)0="" y(treatmen)o(t)c="" (generally)i(outp)q(erforms)f(other)g(approac)o(hes.)0="" fh(3.7)56="" b(pre-pro)r(cessing,)16="" b(in)n(v)m(ariances)i(and)h="" (prior)g(kno)n(wledge)0="" fr(w)l(e)e(ha)o(v)o(e)g(already)g(seen)h="" (that)e(neural)i(net)o(w)o(orks)e(can)h(appro)o(ximate)g(essen)o="" (tially)h(arbitrary)f(nonlinear)h(func-)0="" y(tional)i(mappings)g(b)q="" (et)o(w)o(een)g(sets)g(of)f(v)m(ariables.)34="" b(in)20="" b(principle)j(w)o(e)c(could)i(therefore)e(use)h(a)f(single)i(net)o(w)o="" (ork)0="" y(to)c(transform)f(the)h(ra)o(w)g(input)h(v)m(ariables)h(in)="" o(to)e(the)h(required)g(\014nal)g(outputs.)26="" b(ho)o(w)o(ev)o(er,)17="" b(in)h(practice)g(for)f(all)0="" y(but)f(the)g(simplest)h(problems)g="" (the)f(results)g(of)g(suc)o(h)g(an)g(approac)o(h)g(can)g(b)q(e)h(impro)="" o(v)o(ed)f(up)q(on)g(considerably)i(b)o(y)0="" y(incorp)q(orating)e(v)="" m(arious)g(forms)e(of)h(pre-pro)q(cessing,)h(for)e(reasons)h(whic)o(h)h="" (w)o(e)f(shall)h(outline)h(b)q(elo)o(w.)91="" y(one)e(of)f(the)h="" (simplest)h(and)e(most)g(common)g(forms)g(of)g(pre-pro)q(cessing)i="" (consists)f(of)f(a)g(simple)i(normaliza-)0="" y(tion)g(of)f(the)h="" (input,)h(and)f(p)q(ossibly)h(also)f(target,)e(v)m(ariables.)23="" b(this)16="" b(ma)o(y)f(tak)o(e)g(the)h(form)f(of)h(a)f(linear)i="" (rescaling)0="" y(of)f(eac)o(h)h(input)g(v)m(ariable)h(indep)q(enden)="" o(tly)h(to)d(giv)o(e)h(it)f(zero)h(mean)f(and)h(unit)g(v)m(ariance)g(o)="" o(v)o(er)f(the)g(training)i(set.)0="" y(f)l(or)f(some)h(applications)="" h(the)f(original)h(input)g(v)m(ariables)g(ma)o(y)e(span)h(widely)h="" (di\013eren)o(t)f(ranges.)28="" b(although)18="" b(a)0="" y(linear)i(rescaling)g(of)f(the)g(inputs)h(is)f(equiv)m(alen)o(t)i(to)d="" (a)h(di\013eren)o(t)g(c)o(hoice)h(of)f(\014rst-la)o(y)o(er)f(w)o(eigh)o="" (ts,)h(in)h(practice)0="" y(the)14="" b(optimization)h(algorithm)g(ma)o="" (y)e(ha)o(v)o(e)h(considerable)i(di\016cult)o(y)f(in)g(\014nding)h(a)d="" (satisfactory)h(solution)g(when)0="" y(t)o(ypical)e(input)h(v)m="" (alues)g(are)e(substan)o(tially)i(di\013eren)o(t.)19="" b(similar)13="" b(rescaling)f(can)g(b)q(e)g(applied)i(to)d(the)g(output)h="" (v)m(alues)0="" y(in)18="" b(whic)o(h)f(case)g(the)g(in)o(v)o(erse)g(of)="" f(the)h(transformation)e(needs)j(to)e(b)q(e)h(applied)i(to)d(the)g(net)="" o(w)o(ork)g(outputs)g(when)0="" y(the)f(net)o(w)o(ork)f(is)i(presen)o="" (ted)f(with)g(new)h(inputs.)k(pre-pro)q(cessing)c(is)g(also)f(used)g="" (to)g(enco)q(de)h(data)e(in)i(a)f(suitable)0="" y(form.)22="" b(f)l(or)15="" b(example,)i(if)f(w)o(e)g(ha)o(v)o(e)g(categorical)g(v)m="" (ariables)h(suc)o(h)g(as)e(`red',)h(`green')f(and)h(`blue',)h(these)f="" (ma)o(y)f(b)q(e)0="" y(enco)q(ded)i(using)e(a)g(1-of-3)g(binary)h="" (represen)o(tation.)91="" y(another)f(widely)j(used)e(form)f(of)g="" (pre-pro)q(cessing)i(in)o(v)o(olv)o(es)f(reducing)h(the)e="" (dimensionalit)o(y)j(of)d(the)h(input)0="" y(space.)29="" b(suc)o(h)19="" b(transformations)e(ma)o(y)g(result)i(in)g(loss)f(of)g="" (information)g(in)h(the)f(data,)g(but)h(the)f(o)o(v)o(erall)g(e\013ect)="" y(can)f(b)q(e)h(a)f(signi\014can)o(t)h(impro)o(v)o(emen)o(t)f="" (in)h(p)q(erformance)f(as)g(a)g(consequence)i(of)d(the)i(curse)f(of)g="" (dimensionalit)o(y)0="" y(discussed)22="" b(in)g(section)g(3.5.)35="" b(the)21="" b(\014nite)h(data)e(set)h(is)g(b)q(etter)g(able)h(to)e(sp)q="" (ecify)i(the)f(required)h(mapping)f(in)0="" y(the)d(lo)o(w)o="" (er-dimensional)i(space.)28="" b(dimensionalit)o(y)19="" b(reduction)g(ma)o="" (y)e(b)q(e)i(accomplished)h(b)o(y)d(simply)j(selecting)0="" y(a)g(subset)g(of)f(the)h(original)h(v)m(ariables,)h(but)e(more)f="" (t)o(ypically)j(in)o(v)o(olv)o(es)e(the)g(construction)g(of)g(new)g(v)m="" (ariables)0="" y(consisting)d(of)e(linear)i(or)f(nonlinear)h(com)o="" (binations)f(of)g(the)g(original)h(v)m(ariables)g(called)g="" fq(fe)n(atur)n(es)p="" fr(.)k(a)16="" b(standard)0="" y(tec)o(hnique)22="" b(for)f(dimensionalit)o(y)i(reduction)f(is)g(principal)h(comp)q(onen)o="" (t)e(analysis)h([anderson,)g(1984].)36="" b(suc)o(h)0="" y(metho)q(ds,)13="" b(ho)o(w)o(ev)o(er,)e(mak)o(e)h(use)g(only)h(of)f(the)="" g(input)h(data)f(and)g(ignore)h(the)f(target)f(v)m(alues,)i(and)g(can)f="" (sometimes)0="" y(b)q(e)k(signi\014can)o(tly)h(sub-optimal.)91="" y(y)l(et)k(another)g(form)f(of)h(pre-pro)q(cessing)h(in)o(v)o(olv)="" o(es)g(correcting)f(de\014ciencies)j(in)e(the)g(original)g(data.)37="" y(common)18="" b(o)q(ccurrence)i(is)f(that)f(some)g(of)h(the)f="" (input)i(v)m(ariables)g(are)e(missing)i(for)e(some)g(of)g(the)h(data)f="" (p)q(oin)o(ts.)0="" y(correction)g(of)g(this)h(problem)h(in)f(a)f="" (principled)k(w)o(a)o(y)17="" b(requires)j(that)d(the)i(probabilit)o(y)h="" (distribution)g="" fr(\))d(of)0="" y(input)f(data)f(b)q(e)h(mo)q(deled.)91="" y(one)h(of)f(the)g(most)g="" (imp)q(ortan)o(t)f(factors)h(determining)h(the)g(p)q(erformance)f(of)g="" (real-w)o(orld)h(applications)g(of)0="" y(neural)d(net)o(w)o(orks)f="" (is)h(the)f(use)h(of)f="" fq(prior)j(know)r(le)n(dge)c="" fr(whic)o(h)j(is)f="" (information)f(additional)i(to)e(that)g(presen)o(t)h(in)g(the)0="" y(data.)k(as)13="" b(an)f(example,)i(consider)f(the)g(problem)g(of)f="" (classifying)i(hand-written)f(digits)g(discussed)h(in)f(section)h(1.)0="" y(the)e(most)f(direct)h(approac)o(h)g(w)o(ould)g(b)q(e)g(to)f="" (collect)i(a)e(large)h(training)g(set)g(of)f(digits)i(and)f(to)f(train)="" g(a)h(feedforw)o(ard)952="" y(19)p="" fr(the)17="" b(lik)o(eliho)q(o)q(d)j(function)e(will)h="" (t)o(ypically)g(b)q(e)e(v)o(ery)g(small)h(except)f(for)g(v)m(alues)h="" (of)f="" fn(w)g="" fr(for)g(whic)o(h)h(the)f(net)o(w)o(ork)0="" y(function)11="" b(is)h(reasonably)e(consisten)o(t)h(with)g(the)g="" (data.)17="" b(th)o(us)11="" b(the)f(p)q(osterior)h(distribution)h="" fk(jd)q="" fr(\))d(will)k(b)q(e)e(m)o(uc)o(h)0="" y(more)16="" b(sharply)h(p)q(eak)o(ed)h(than)e(the)h(prior)g="" (distribution)h="" fr(\))d(\(and)i(will)h(t)o="" (ypically)g(ha)o(v)o(e)e(m)o(ultiple)j(maxima\).)0="" y(the)f(quan)o(tit)o(y)e(w)o(e)h(are)g(in)o(terested)h(in)g(is)g(the)f="" (predicted)i(distribution)g(of)e(target)f(v)m(alues)i="" fn(t)g="" fr(for)e(a)h(new)h(input)0="" y(v)o(ector)e="" fn(x)g="" fr(once)g(w)o(e)g(ha)o(v)o(e)g(observ)o(ed)h(the)f(data)g(set)g="" fk(d)q="" fr(.)23="" b(this)17="" b(can)g(b)q(e)g(expressed)g(as)f(an)g(in)o="" (tegration)g(o)o(v)o(er)g(the)0="" y(p)q(osterior)f(distribution)i(of)="" e(w)o(eigh)o(ts)g(of)f(the)i(form:)636="" fr(\))j(=")872" ff(z)921="" fm(;)d="" fr(\))p="" fk(j)o(d)q="" fr(\))g="" fm(d)o="" fn(w)553="" fr(\(30\))0="" fr(\))13="" b(is)j(the)f(conditional)i(probabilit)o(y)f(mo)q(del)g="" (discussed)h(in)f(the)f(in)o(tro)q(duction.)91="" y(if)h(w)o(e)g(supp)="" q(ose)g(that)g(the)g(p)q(osterior)f(distribution)j="" fr(\))d(is)h(sharply)g(p)q(eak)o(ed)h(around)f="" (a)f(single)j(most-)0="" y(probable)e(v)m(alue)g="" fn(w)342="" fj(mp)401="" fr(,)e(then)i(w)o(e)f(can)g(write)g(eq.)g(30)g="" (in)h(the)f(form:)575="" fr(\))39="" fk(')j="" fn(w)1029="" fj(mp)1086="" fr(\))1112="" ff(z)1160="" fm(d)p="" fn(w)492="" fr(\(31\))791="" y(=")42" fr(\))765="" b(\(32\))0="" y(and)17="" b(so)f(predictions)i(can)f(b)q(e)g(made)f(b)o(y)h="" (\014xing)g(the)g(w)o(eigh)o(ts)f(to)g(their)h(most)f(probable)h(v)m="" (alues.)25="" b(w)l(e)17="" b(can)f(\014nd)0="" y(the)e(most)f(probable)i="" (w)o(eigh)o(ts)e(b)o(y)h(maximizing)h(the)f(p)q(osterior)g="" (distribution,)h(or)e(equiv)m(alen)o(tly)k(b)o(y)c(minimizing)0="" y(its)j(negativ)o(e)g(logarithm.)22="" b(using)17="" b(eq.)e(29,)g(w)o="" (e)h(see)g(that)f="" fn(w)1036="" fj(mp)1110="" fr(is)h(determined)i(b)o(y)d(minimizing)k(a)d(regularized)0="" y(cost)f(function)i(of)e(the)h(form)f(in)i(eq.)e(27)h(in)g(whic)o="" (h)h(the)f(negativ)o(e)g(log)g(of)f(the)h(prior)g="" fk(\000)8="" fr(ln)h="" fr(\))14="" b(represen)o(ts)i(the)0="" y(regularizer)g="" fm(\027)s="" fr(\012.)21="" b(example,)g(if)h(the)g(prior)f(consists)h(of)f(a)g(zero-mean)g="" (gaussian)h(with)f(v)m(ariance)i="" fm(\027)1799="" fj(1)1862="" fr(then)0="" y(w)o(e)e(obtain)g(the)h(w)o="" (eigh)o(t-deca)o(y)f(regularizer)h(of)f(eq.)f(28.)91="" y(the)k(p)q(osterior)f(distribution)i(will)h(b)q(ecome)e(sharply)g="" (p)q(eak)o(ed)g(when)g(the)g(size)g(of)f(the)h(data)f(set)g(is)h(large)="" y(compared)i(to)f(the)i(n)o(um)o(b)q(er)f(of)g(parameters)f(in)i="" (the)f(net)o(w)o(ork.)33="" b(f)l(or)20="" b(data)f(sets)h(of)f(limited)j="" (size,)g(ho)o(w)o(ev)o(er,)0="" y(the)d(p)q(osterior)h(distribution)h="" (has)e(a)g(\014nite)h(width)g(and)f(this)h(adds)g(to)e(the)i(uncertain)="" o(t)o(y)f(in)h(the)f(predictions)0="" y(for)e="" fn(t)i="" fr(whic)o(h)g(can)f(b)q(e)g(expressed)h(in)g(terms)e(of)h(error)f="" (bars.)28="" b(ba)o(y)o(esian)18="" b(error)g(bars)f(can)h(b)q(e)h(ev)m="" (aluated)g(using)0="" y(a)i(lo)q(cal)h(gaussian)f(appro)o(ximation)g="" (to)f(the)h(p)q(osterior)g(distribution)i([macka)o(y)l(,)e(1992].)36="" b(presence)h(of)0="" y(m)o(ultiple)c(maxima)e(in)i(the)e(p)q="" (osterior)h(distribution)h(also)e(con)o(tributes)h(to)f(the)g="" (uncertain)o(ties)i(in)f(predictions.)0="" y(the)e(capabilit)o(y)i="" (to)e(assess)f(these)i(uncertain)o(ties)g(can)f(pla)o(y)h(a)f(crucial)h="" (role)g(in)g(practical)g(applications.)91="" y(the)f(ba)o(y)o(esian)g="" (approac)o(h)g(can)g(also)g(deal)g(with)h(more)e(general)i(problems)f="" (in)h(complexit)o(y)g(con)o(trol.)j(this)0="" y(can)c(b)q(e)h(done)g="" (b)o(y)f(considering)i(the)e(probabilities)i(of)e(a)g(set)g(of)g="" (alternativ)o(e)g(mo)q(dels,)h(giv)o(en)g(the)f(data)f(set:)718="" fk(h)797="" fi(i)811="" fr(\))f(=")944" fk(d)q(jh)1072="" fi(i)1086="" fk(h)1183="" fi(i)1196="" fr(\))1219="" fm(:)637="" fr(\(33\))0="" y(here)11="" b(di\013eren)o(t)g(mo)q(dels)g(can)g(also)="" g(b)q(e)g(in)o(terpreted)h(as)e(di\013eren)o(t)h(v)m(alues)h(of)e="" (regularization)i(parameters)d(as)i(these)0="" y(to)q(o)j(con)o(trol)="" f(mo)q(del)j(complexit)o(y)l(.)k(if)15="" b(the)f(mo)q(dels)h(are)f(giv)o="" (en)g(the)h(same)f(prior)g(probabilities)i="" fk(h)1714="" fi(i)1729="" fr(\))d(then)i(they)0="" y(can)d(b)q(e)h(rank)o(ed)g(b)o(y)f(considering)i(the)e="" fq(evidenc)n(e)f="" fk(d)q(jh)961="" fi(i)975="" fr(\))h(whic)o(h)h(itself)h(can)e(b)q(e)h(ev)m(aluated)="" g(b)o(y)g(in)o(tegration)f(o)o(v)o(er)0="" y(the)h(mo)q(del)g="" (parameters)f="" b(w)l(e)13="" b(can)f(simply)i(select)g(the)f="" (mo)q(del)g(with)g(the)g(greatest)f(probabilit)o(y)l(.)20="" b(ho)o(w)o(ev)o(er,)12="" y(full)j(ba)o(y)o(esian)e(treatmen)o="" (t)f(requires)i(that)f(w)o(e)g(form)f(a)h(linear)i(com)o(bination)f(of)="" f(the)g(predictions)i(of)e(the)g(mo)q(dels)0="" y(in)j(whic)o(h)g="" (the)f(w)o(eigh)o(ting)h(co)q(e\016cien)o(ts)g(are)f(giv)o(en)h(b)o(y)f="" (the)g(mo)q(del)h(probabilities.)91="" y(in)g(general,)f(the)g="" (required)i(in)o(tegrations,)e(suc)o(h)g(as)g(that)g(in)h(eq.)e(30,)h="" (are)g(analytically)i(in)o(tractable.)j(one)0="" y(approac)o(h)e(is)h="" (to)f(appro)o(ximate)g(the)g(p)q(osterior)g(distribution)j(b)o(y)d(a)g="" (gaussian)g(cen)o(tered)h(on)g="" fn(w)1694="" fj(mp)1770="" fr(and)g(then)952="" y(18)p="" fr(c)o(hosen)14="" b(to)g(encourage)g(smo)q(oth)f="" (functions.)21="" b(the)14="" b(simplest)h(example)g(is)g(called)g="" fq(weight)h(de)n(c)n(ay)e="" fr(and)g(consists)g(of)0="" y(the)h(sum)g(of)g(the)h(squares)f(of)f(all)i(the)g(adaptiv)o(e)f="" (parameters)f(in)i(the)g(mo)q(del:)868="" y(\012)c(=")961" ff(x)985="" fi(i)1029="" fm(w)1063="" fj(2)1062="" fi(i)1869="" fr(\(28\))0="" y(consider)19="" b(the)f(e\013ect)h(of)f(suc)o(h)g(a)g(term)g(on)h(the)f(mlp)h(function)="" g(\(eq.)e(9\).)29="" b(if)19="" b(the)f(w)o(eigh)o(ts)g(tak)o(e)g(v)o(ery)g="" (small)0="" y(v)m(alues)i(then)f(the)g(net)o(w)o(ork)f(outputs)g(b)q="" (ecome)i(appro)o(ximately)f(linear)h(functions)f(of)g(the)f(inputs)i="" (\(since)g(the)0="" y(sigmoidal)h(function)g(is)f(appro)o(ximately)g="" (linear)h(for)e(small)i(v)m(alues)g(of)e(its)i(argumen)o(t\).)32="" b(the)20="" b(v)m(alue)i(of)d="" fm(\027)k="" fr(in)0="" y(eq.)12="" b(27)f(con)o(trols)h(the)h(e\013ectiv)o(e)f(complexit)o(y)h(of)f(the)g="" (mo)q(del,)i(so)e(that)f(for)h(large)g="" fr(the)c(mo)q(del)h="" (is)g(o)o(v)o(er-smo)q(othed)0="" y(\(corresp)q(onding)e(to)f(high)h="" (bias\))g(while)g(for)f(small)h="" fm(\027)j="" fr(the)c(mo)q(del)i(can)e(o)o="" (v)o(er\014t)g(\(corresp)q(onding)h(to)e(high)j(v)m(ariance\).)0="" y(w)l(e)i(can)f(therefore)h(consider)g(a)f(net)o(w)o(ork)g(with)h="" (a)f(relativ)o(ely)i(large)e(n)o(um)o(b)q(er)h(of)f(hidden)j(units)e="" (and)g(con)o(trol)f(the)0="" y(e\013ectiv)o(e)i(complexit)o(y)h(b)o(y)="" f(c)o(hanging)h="" fr(.)k(in)c(practice,)f(a)g(suitable)h(v)m="" (alue)h(for)d="" fr(can)e(b)q(e)f(found)h(b)o(y)f(seeking)h(the)="" y(v)m(alue)g(whic)o(h)g(giv)o(es)g(the)f(b)q(est)h(p)q="" (erformance)f(on)g(a)g(v)m(alidation)i(set.)91="" y(the)i(w)o(eigh)o="" (t)f(deca)o(y)h(regularizer)h(\(eq.)e(28\))g(is)h(simple)h(to)e="" (implemen)o(t)j(but)d(su\013ers)h(from)f(a)g(n)o(um)o(b)q(er)h(of)0="" y(limitations.)34="" b(regularizers)21="" b(used)f(in)g(practice)h(ma)o="" (y)d(b)q(e)i(more)g(sophisticated)g(and)g(ma)o(y)e(con)o(tain)i(m)o="" (ultiple)0="" y(regularization)c(co)q(e\016cien)o(ts)g([neal,)f="" (1994].)91="" y(regularization)e(metho)q(ds)e(can)h(b)q(e)g="" (justi\014ed)h(within)f(a)f(general)h(theoretical)g(framew)o(ork)e(kno)="" o(wn)i(as)f="" fq(struc-)0="" y(tur)n(al)17="" b(risk)f(minimization)j="" fr([v)l(apnik,)d(1995].)21="" b(structural)16="" b(risk)g(minimization)i(pro)="" o(vides)e(a)f(quan)o(titativ)o(e)h(mea-)0="" y(sure)e(of)f(complexit)="" o(y)i(kno)o(wn)f(as)f(the)h="" fq(v)o(c)g(dimension)p="" fr(.)k(the)d(theory)="" e(sho)o(ws)g(that)g(the)h(v)o(c)g(dimension)h(predicts)0="" y(the)g(di\013erence)h(b)q(et)o(w)o(een)f(p)q(erformance)g(on)g(a)="" g(training)g(set)g(and)g(p)q(erformance)g(on)f(a)h(test)f(set;)h(th)o="" (us,)f(the)h(sum)0="" y(of)i(log)h(lik)o(eliho)q(o)q(d)i(and)e="" (\(some)f(function)i(of)t(\))d(v)o(c)i(dimension)h(pro)o(vides)f(a)f="" (measure)h(of)f(generalization)i(p)q(er-)0="" y(formance.)g(this)d="" (motiv)m(ates)f(regularization)h(metho)q(ds)f(\(eq.)g(27\))f(and)h(pro)="" o(vides)h(some)e(insigh)o(t)i(in)o(to)f(p)q(ossible)0="" y(forms)f(for)h(the)g(regularizer)h(\012.)0="" fh(3.6)56="" b(ba)n(y)n(esian)18="" b(viewp)r(oin)n(t)0="" fr(in)h(earlier)g(sections)f(w)o(e)g(discussed)i(net)o(w)o(ork)d="" (training)i(in)g(terms)e(of)h(the)g(minimization)i(of)e(a)g(cost)f="" (function)0="" y(deriv)o(ed)h(from)f(the)g(principle)j(of)d(maxim)o="" (um)g(a)g(p)q(osteriori)h(or)f(maxim)o(um)g(lik)o(eliho)q(o)q(d)j="" (estimation.)26="" b(this)18="" b(ap-)0="" y(proac)o(h)j(can)g(b)q(e)h="" (seen)g(as)e(a)h(particular)h(appro)o(ximation)f(to)g(a)g(more)f="" (fundamen)o(tal,)j(and)e(more)g(p)q(o)o(w)o(erful,)0="" y(framew)o(ork)16="" b(based)i(on)f(ba)o(y)o(esian)h(statistics.)26="" b(in)18="" b(the)g(maxim)o(um)f(lik)o(eliho)q(o)q(d)k(approac)o(h)c(the)g="" (w)o(eigh)o(ts)g="" fn(w)h="" fr(are)0="" y(set)c(to)f(a)h(sp)q(eci\014c)h="" (v)m(alue)h="" fn(w)472="" fj(ml)542="" fr(determined)f(b)o(y)f="" (minimization)i(of)d(a)h(cost)f(function.)21="" b(ho)o(w)o(ev)o(er,)13="" b(w)o(e)g(kno)o(w)h(that)0="" y(there)19="" b(will)h(t)o(ypically)g(b)q="" (e)g(other)e(minima)i(of)e(the)h(cost)f(function)h(whic)o(h)h(migh)o(t)="" e(giv)o(e)h(equally)h(go)q(o)q(d)f(results.)0="" y(also,)d(w)o(eigh)o="" (t)g(v)m(alues)h(close)f(to)g="" fn(w)601="" fj(ml)673="" fr(should)h(giv)o(e)f(results)g(whic)o(h)h(are)f(not)f(to)q(o)g="" (di\013eren)o(t)i(from)e(those)g(of)h(the)0="" y(maxim)o(um)f(lik)o="" (eliho)q(o)q(d)j(w)o(eigh)o(ts)d(themselv)o(es.)91="" y(these)20="" b(e\013ects)g(are)g(handled)i(in)f(a)e(natural)i(w)o(a)o(y)e="" (in)i(the)f(ba)o(y)o(esian)g(viewp)q(oin)o(t,)i(whic)o(h)f(describ)q="" (es)h(the)0="" y(w)o(eigh)o(ts)e(not)h(in)g(terms)f(of)g(a)h(sp)q="" (eci\014c)h(set)f(of)f(v)m(alues,)j(but)d(in)i(terms)e(of)g(a)h="" (probabilit)o(y)g(distribution)i(o)o(v)o(er)0="" y(all)e(p)q(ossible)="" i(v)m(alues.)37="" b(as)20="" b(discussed)i(earlier)g(\(cf.)35="" b(eq.)20="" b(13\),)h(once)f(w)o(e)h(observ)o(e)f(the)g(training)i(data)d="" (set)i="" fk(d)0="" fr(w)o(e)c(can)h(compute)g(the)f(corresp)q(onding)="" i="" fq(p)n(osterior)e="" fr(distribution)j(using)e(ba)o(y)o(es')e(theorem,)i="" (based)g(on)f(a)g="" fq(prior)0="" fr(distribution)g(function)f="" fr(\))e(\(whic)o(h)h(will)i(t)o(ypically)g(b)q(e)="" f(v)o(ery)f(broad\),)f(and)h(a)g="" fq(likeliho)n(o)n(d)g="" fr(function)h="" fk(d)q(j)p="" fr(\):)739="" fr(\))c(=")951" fr(\))1198="" fm(:)658="" fr(\(29\))952="" y(17)p="" fr(dashed)15="" b(curv)o(es\).)k(this)d(data)d(set)i="" (has)f(then)h(b)q(een)g(\014tted)g(b)o(y)f(a)h(mixture)f(of)g="" fm(m)20="" fr(gaussians)14="" b(b)o(y)h(use)g(of)f(the)g(em)0="" y(algorithm.)19="" b(w)l(e)c(see)f(that)f(a)h(mo)q(del)h(with)f(1)g="" (comp)q(onen)o(t)g(\()p="" fm(m)j="" (represen)o(tation)g(of)g(the)g(true)0="" y(distribution)h(from)e="" (whic)o(h)h(the)g(data)e(w)o(as)h(generated,)g(and)h(in)g(particular)g="" (is)g(unable)h(to)e(capture)g(the)h(bimo)q(dal)0="" y(asp)q(ect.)19="" b(f)l(or)13="" fm(m)k="" (es)f(a)g(go)q(o)q(d)f(\014t,)h(as)g(w)o(e)g(exp)q(ect)g(since)h(the)f="" (data)g(w)o(as)f(itself)i(generated)f(from)0="" y(a)18="" b(t)o(w)o(o-comp)q(onen)o(t)f(gaussian)h(mixture.)28="" b(increasing)j(the)e(n)o(um)o(b)q(er)g(of)f="" (comp)q(onen)o(ts)h(to)g="" y(giv)o(es)d(a)g(p)q(o)="" q(orer)g(\014t,)g(ev)o(en)h(though)f(this)g(mo)q(del)h(con)o(tains)g="" (the)f(simpler)h(mo)q(dels)h(as)d(sp)q(ecial)j(cases.)91="" y(the)11="" b(problem)g(is)g(a)f(v)o(ery)g(fundamen)o(tal)h(one)g(and)="" f(is)i(asso)q(ciated)e(with)h(the)g(fact)f(that)f(w)o(e)i(are)f(trying)="" g(to)g(infer)0="" y(an)j(en)o(tire)h(distribution)h(function)f(from)e="" (a)h(\014nite)h(n)o(um)o(b)q(er)g(of)f(data)f(p)q(oin)o(ts,)i(whic)o(h)="" g(is)g(necessarily)g(an)f(ill-p)q(osed)0="" y(problem.)20="" b(in)c(regression)f(for)f(example)i(there)f(are)g(in\014nitely)i(man)o="" (y)e(functions)g(whic)o(h)h(will)h(giv)o(e)e(a)g(p)q(erfect)g(\014t)0="" y(to)g(the)h(\014nite)g(n)o(um)o(b)q(er)g(of)g(data)f(p)q(oin)o="" (ts.)21="" b(if)16="" b(the)g(data)f(are)h(noisy)l(,)g(ho)o(w)o(ev)o(er,)e="" (the)i(b)q(est)g(generalization)h(will)g(b)q(e)0="" y(obtained)i(for)f(a)h(function)g(whic)o(h)g(do)q(es)g(not)g(\014t)f="" (the)h(data)f(p)q(erfectly)h(but)g(whic)o(h)h(captures)e(the)h="" (underlying)0="" y(function)14="" b(from)f(whic)o(h)h(the)f(data)g(w)o="" (ere)g(generated.)19="" b(by)14="" b(increasing)g(the)g(\015exibilit)o(y)i="" (of)d(the)g(mo)q(del)h(w)o(e)f(are)g(able)0="" y(to)g(obtain)h(ev)o="" (er)g(b)q(etter)g(\014ts)f(to)h(the)g(training)g(data,)f(and)h(this)g="" (is)h(re\015ected)f(in)h(a)f(steadily)g(increasing)i(v)m(alue)f(for)0="" y(the)g(lik)o(eliho)q(o)q(d)i(function)f(at)e(its)g(maxim)o(um.)20="" b(our)15="" b(goal)f(is)i(to)e(mo)q(del)h(the)g(true)f(underlying)j="" (densit)o(y)e(function)0="" y(from)f(whic)o(h)i(the)f(data)f(w)o(as)g="" (generated)h(since)h(this)g(allo)o(ws)f(us)g(to)f(mak)o(e)h(the)g(b)q="" (est)g(predictions)h(for)f(new)g(data.)0="" y(w)l(e)g(see)h(that)e="" (the)i(b)q(est)f(appro)o(ximation)g(to)g(this)g(densit)o(y)h(o)q(ccurs)="" g(for)e(an)h(in)o(termediate)h(v)m(alue)h(of)e="" fm(m)5="" fr(.)91="" b(same)f(issue)h(arises)g(in)g(connection)h="" (with)e(nonlinear)i(regression)f(and)f(classi\014cation)i(problems.)21="" b(f)l(or)0="" y(example,)e(the)f(n)o(um)o(b)q(er)h="" fr(of)17="" b(hidden)j(units)f(in)f(an)g(mlp)g(net)o(w)o(ork)f(con)="" o(trols)h(the)g(mo)q(del)h(complexit)o(y)g(and)0="" y(m)o(ust)13="" b(b)q(e)i(optimized)h(to)d(giv)o(e)h(the)g(b)q(est)h="" (generalization.)20="" b(in)15="" b(a)f(practical)g(application)i(w)o(e)e="" (can)g(train)g(a)g(v)m(ariet)o(y)0="" y(of)e(di\013eren)o(t)g(mo)q="" (dels)h(ha)o(ving)f(di\013eren)o(t)g(complexit)o(y)l(,)i(and)e(compare)="" g(their)h(generalization)g(p)q(erformance)f(using)0="" y(an)k(indep)q(enden)o(t)i(v)m(alidation)f(set,)e(and)h(then)g(select)h="" (the)e(mo)q(del)i(with)f(the)g(b)q(est)g(generalization.)22="" b(in)17="" b(fact)e(the)0="" y(pro)q(cess)h(of)g(optimizing)i(the)e="" (complexit)o(y)h(using)f(a)g(v)m(alidation)i(set)e(can)g(lead)h(to)e="" (some)h(partial)g(o)o(v)o(er\014tting)g(to)0="" y(the)g(v)m="" (alidation)i(data)e(itself,)h(and)f(so)g(the)g(\014nal)h(p)q="" (erformance)g(of)e(the)i(selected)g(mo)q(del)g(should)h(b)q(e)e="" (con\014rmed)0="" y(using)g(a)f(third)h(indep)q(enden)o(t)h(data)e="" (set)g(called)h(a)f="" fq(test)g="" fr(set.)91="" y(some)i(theoretical)g="" (insigh)o(t)h(in)o(to)e(the)h(problem)g(of)g(o)o(v)o(er\014tting)f(can)="" h(b)q(e)g(obtained)g(b)o(y)g(decomp)q(osing)h(the)0="" y(error)13="" b(in)o(to)g(the)g(sum)h(of)e="" fq(bias)h="" fq(varianc)n(e)e="" fr(terms)h([geman,)g(et)g(al.,)g(1992].)18="" b(mo)q(del)h(whic)o(h)h(is)e(to)q(o)g(in\015exible)0="" y(is)18="" b(unable)g(to)f(represen)o(t)g(the)g(true)g(structure)g="" (in)h(the)g(underlying)h(densit)o(y)e(function)i(and)e(this)h(giv)o(es)="" f(rise)h(to)0="" y(a)e(high)i(bias.)24="" b(con)o(v)o(ersely)16="" b(a)h(mo)q(del)g(whic)o(h)g(is)h(to)q(o)d(\015exible)k(b)q(ecomes)e="" (tuned)g(to)f(the)h(sp)q(eci\014c)h(details)g(of)e(the)0="" y(particular)d(data)e(set)h(and)g(giv)o(es)g(a)f(high)i(v)m="" (ariance.)20="" b(the)12="" b(b)q(est)g(generalization)i(is)e(obtained)h="" (from)e(the)h(optim)o(um)0="" y(trade-o\013)i(of)h(bias)h(against)e="" (v)m(ariance.)91="" y(as)i(w)o(e)f(ha)o(v)o(e)h(already)g(remark)o="" (ed,)f(the)h(problem)h(of)f(inferring)h(an)e(en)o(tire)i(distribution)g="" (function)g(from)e(a)0="" y(\014nite)20="" b(data)e(set)g(is)h(fundamen)="" o(tally)h(ill-p)q(osed)i(since)d(there)g(are)g(in\014nitely)i(man)o(y)d="" (solutions.)31="" b(the)19="" b(problem)0="" y(only)g(b)q(ecomes)h(w)o="" (ell-p)q(osed)h(when)e(some)g(additional)h(constrain)o(t)f(is)g(imp)q="" (osed.)32="" b(this)20="" b(constrain)o(t)e(migh)o(t)h(b)q(e)0="" y(that)12="" b(w)o(e)g(mo)q(del)i(the)f(data)f(using)h(a)g(net)o(w)o="" (ork)e(ha)o(ving)i(a)g(limited)h(n)o(um)o(b)q(er)f(of)f(hidden)j="" (units.)20="" b(within)14="" b(the)e(range)0="" y(of)19="" b(functions)i(whic)="" o(h)f(this)h(mo)q(del)f(can)g(represen)o(t)g(there)g(is)g(then)g(a)g="" (unique)h(function)g(whic)o(h)f(b)q(est)g(\014ts)g(the)0="" y(data.)26="" b(implicitly)20="" b(w)o(e)d(are)g(assuming)h(that)e(the)i="" (underlying)h(densit)o(y)f(function)g(from)f(whic)o(h)h(the)g(data)e(w)="" o(ere)0="" y(dra)o(wn)i(is)i(relativ)o(ely)g(smo)q(oth.)30="" b(instead)19="" b(of)g(limiting)i(the)e(n)o(um)o(b)q(er)g(of)f(parameters)="" g(in)i(the)f(mo)q(del,)h(w)o(e)f(can)0="" y(encourage)d(smo)q="" (othness)g(more)g(directly)i(using)f(the)f(tec)o(hnique)i(of)e="" fq(r)n(e)n(gularization)p="" fr(.)22="" b(in)o(v)o(olv)o(es)g="" (adding)g(a)0="" y(p)q(enalt)o(y)f(term)f(\012)g(to)f(the)h(original)="" i(cost)d(function)i="" fm(j)k="" fr(to)15="" b(giv)o(e)g(a)g(total)g(cost)f="" (function)1512="" ff(e)1501="" fm(j)20="" fr(of)15="" b(the)g(form:)869="" ff(e)858="" fm(j)i="" fr(+)c="" fr(\012)777="" b(\(27\))0="" y(where)11="" fr(is)c(called)i(a)e(regularization)h(co)q(e\016cien)o(t.)="" b(net)o(w)o(ork)e(parameters)g(are)h(determined)h(b)o(y)f="" (minimizing)11="" ff(e)0="" fm(j)5="" fr(,)11="" b(and)h(the)f(v)m(alue)h(of)f="" fr(con)o(trols)c(the)g(degree)h="" (of)e(in\015uence)k(of)d(the)g(p)q(enalt)o(y)h(term)f(\012.)18="" b(in)12="" b(practice)g(\012)f(is)h(t)o(ypically)952="" y(16)p="" starttexfig="" %%begindocument:="" ps="" em.eps="" mathworks="" dict="" begin="" bdef="" {bind="" def}="" bind="" def="" ldef="" {load="" xdef="" {exch="" xstore="" store}="" c="" clip="" cc="" concat="" cp="" closepath="" gr="" grestore="" gs="" gsave="" mt="" moveto="" np="" newpath="" cm="" currentmatrix="" sm="" setmatrix="" rc="" {rectclip}="" rf="" {rectfill}="" rm="" rmoveto="" rl="" rlineto="" s="" show="" sc="" {setcmykcolor}="" sr="" setrgbcolor="" w="" setlinewidth="" cap="" setlinecap="" pgsv="" ()="" bpage="" {="" save="" epage="" {pgsv="" restore}="" bplot="" eplot="" {stroke="" grestore}="" portraitmode="" landscapemode="" dpi2point="" fontsize="" fms="" %save="" size="" off="" stack="" findfont="" [fontsize="" neg="" 0]="" makefont="" setfont="" }bdef="" reencode="" exch="" dup="" where="" {pop="" load}="" standardencoding}="" ifelse="" roll="" length="" index="" fid="" ne="" {def}{pop="" pop}="" }="" forall="" encoding="" currentdict="" end="" definefont="" pop="" isroman="" charstrings="" get="" agrave="" known="" fmsr="" {reencode}="" csm="" div="" -1="" scale="" translate="" eq="" {90="" rotate}="" if="" so="" []="" setdash="" do="" [.5="" mul="" mul]="" da="" [6="" dd="" l="" lineto="" stroke="" mp="" sub="" {rlineto}="" repeat="" ap="" pp="" fill="" dp="" mr="" -2="" fr="" pr="" l1i="" currentfile="" picstr="" readhexstring="" image="" tmatrix="" matrix="" makeoval="" arc="" fo="" po="" pd="" copy="" setlinejoin="" 0216="" %colortable="" dictionary="" c0="" sr}="" c1="" c2="" c3="" c4="" c5="" c6="" c7="" helvetica="" isolatin1encoding="" (0.0)="" (0.5)="" (1.0)="" (3.0)="" -3="" -4="" -5="" times-roman="" (1)="" times-italic="" (m=")" (2)="" -32="" -63="" -86="" -99="" -100="" -92="" -79="" -62="" -46="" -33="" -21="" -14="" -8="" -13="" -17="" -25="" -27="" -30="" -31="" -29="" -23="" -20="" -18="" -15="" -11="" -6="" -7="" -9="" -10="" -16="" -22="" -24="" -28="" -19="" -12="" (10)="" showpage="" %%enddocument="" endtexfig="" fr(figure)16="" b(4:)21="" b(e\013ects)15="" b(of)g(mo)q(del)i="" (complexit)o(y)f(illustrated)i(b)o(y)d(mo)q(deling)j(a)d(mixture)h(of)f="" (t)o(w)o(o)g(gaussians)h(\(sho)o(wn)0="" y(b)o(y)h(the)h(dashed)g="" (curv)o(es\))g(using)g(a)f(mixture)h(of)f="" fm(m)23="" fr(gaussians)17="" b(\(sho)o(wn)g(b)o(y)h(the)f(solid)i(curv)o(es\).)27="" b(the)18="" b(results)0="" y(are)d(obtained)h(for)e(20)h(cycles)h(of)f="" (em.)952="" y(15)p="" fr(the)14="" b(algorithms)f(discussed)j(so)d(far)g(are)="" g(called)i="" fq(b)n(atch)f="" fr(since)h(they)f(in)o(v)o(olv)o(e)g(using)g="" (the)g(whole)g(data)f(set)h(for)0="" y(eac)o(h)h(ev)m(aluation)i(of)e="" (the)g(cost)g(function)h(or)f(its)g(gradien)o(t.)20="" b(there)c(is)g="" (also)f(a)g="" fq(sto)n(chastic)g="" fr(or)g="" fq(on-line)f="" fr(v)o(ersion)h(of)="" y(gradien)o(t)f(descen)o(t)g(in)g(whic)o(h,)h(for)e(eac)o(h)h="" (parameter)f(up)q(date,)h(the)g(cost)f(function)h(gradien)o(t)g(is)g="" (ev)m(aluated)h(using)0="" y(just)j(one)h(of)e(the)i(training)g(v)o="" (ectors)e(at)h(a)g(time)h(\(whic)o(h)g(are)f(then)g(cycled)i(either)f="" (in)g(order)f(or)g(in)h(a)f(random)0="" y(sequence\).)29="" b(while)19="" b(this)g(approac)o(h)e(fails)i(to)f(mak)o(e)f(use)h(of)g="" (the)g(p)q(o)o(w)o(er)f(of)h(sophisticated)h(metho)q(ds)f(suc)o(h)g(as)="" y(conjugate)13="" b(gradien)o(ts,)g(it)h(can)f(pro)o(v)o(e)g="" (e\013ectiv)o(e)g(for)g(v)o(ery)g(large)g(data)g(sets,)g(particularly)h="" (if)g(there)f(is)h(signi\014can)o(t)0="" y(redundancy)i(in)g(the)g="" (data.)0="" fh(3.4)56="" b(hessian)18="" b(matrices,)e(error)i(bars)h(and)="" g(pruning)0="" fr(after)c(a)g(set)g(of)g(w)o(eigh)o(ts)g(ha)o(v)o(e)="" g(b)q(een)h(found)g(for)f(a)g(neural)h(net)o(w)o(ork)e(using)i(an)g="" (optimization)g(pro)q(cedure,)g(it)f(is)0="" y(often)j(useful)h(to)e="" (examine)i(second-order)g(prop)q(erties)f(of)g(the)g(\014tted)g(net)o="" (w)o(ork)f(as)h(captured)g(in)h(the)f(hessian)0="" y(matrix)12="" fm(h)k="" fm(@)274="" fj(2)293="" fm(j)r(="@)s" fm(@)s="" fn(w)475="" fi(t)501="" b(e\016cien)o(t)13="" b(algorithms)f(ha)o(v)o(e)g(b)q(een)h(dev)o(elop)q(ed)h(to)e(compute)g="" (the)g(hessian)h(matrix)0="" y(in)g(time)g="" fm(o)q="" fm(w)254="" fj(2)274="" fr(\))f([bishop,)h(1995].)k(as)c(in)g="" (the)g(case)f(of)g(the)h(calculation)h(of)e(the)g(gradien)o(t)h(b)o(y)f="" (bac)o(kpropagation,)0="" y(these)j(algorithms)h(are)e(based)i(on)f="" (recursiv)o(e)h(message)f(passing)g(in)h(the)g(net)o(w)o(ork.)91="" y(one)i(imp)q(ortan)o(t)f(use)g(of)g(the)g(hessian)h(matrix)f="" (lies)h(in)g(the)f(calculation)i(of)e(error)f(bars)h(on)g(the)g="" (outputs)0="" y(of)g(a)h(net)o(w)o(ork.)27="" b(if)18="" b(w)o(e)g(appro)o(ximate)f(the)h(cost)g(function)g(lo)q(cally)i(as)d(a)="" h(quadratic)g(function)h(of)e(the)h(w)o(eigh)o(ts)0="" y(\(an)13="" b(appro)o(ximation)h(whic)o(h)g(is)g(equiv)m(alen)o(t)h(to)e="" (making)h(a)f(gaussian)h(appro)o(ximation)g(for)f(the)g(log)h(lik)o="" (eliho)q(o)q(d\),)0="" y(then)i(the)f(estimated)g(v)m(ariance)h(of)f="" (the)g="" fm(i)713="" fi(th)763="" fr(output)g="" fm(y)934="" fi(i)964="" fr(can)g(b)q(e)h(sho)o(wn)f(to)f(b)q(e:)702="" y(^)-25="" fm(\033)728="" fj(2)726="" fi(y)743="" fd(i)771="" ff(\022)855="" fm(@)s(y)904="" fn(w)925="" ff(\023)955="" fi(t)990="" fm(h)1032="" fj(1)1086="" ff(\022)1123="" fm(@)s(y)1172="" fn(w)1192="" ff(\023)1230="" fm(;)626="" fr(\(26\))0="" b(the)h(gradien)o(t)f(v)o(ector)f="" fm(@)s(y)573="" fi(i)587="" fm(="@)s" fr(can)h(b)q(e)g="" (calculated)g(via)g(bac)o(kpropagation.)91="" y(the)f(hessian)h="" (matrix)f(is)g(also)g(useful)h(in)g(pruning)g(algorithms.)k(a)15="" b(pruning)h(algorithm)f(deletes)h(w)o(eigh)o(ts)0="" y(from)f(a)h(\014tted)f(net)o(w)o(ork)g(to)g(yield)j(a)d(simpler)i(net)="" o(w)o(ork)e(that)g(ma)o(y)g(outp)q(erform)g(a)h(more)f(complex,)i(o)o="" (v)o(er\014tted)0="" y(net)o(w)o(ork)c(\(see)h(b)q(elo)o(w\),)g(and)g="" (ma)o(y)g(b)q(e)g(easier)h(to)e(in)o(terpret.)20="" b(this)f(setting,)g(the)g(hessian)h(is)f(used)h(to)e(appro)o(x-)0="" y(imate)j(the)h(increase)g(in)g(the)f(cost)g(function)h(due)f(to)g="" (the)g(deletion)i(of)e(a)f(w)o(eigh)o(t.)23="" b(a)16="" b(v)m(ariet)o(y)h="" (of)e(suc)o(h)i(pruning)0="" y(algorithms)e(are)g(a)o(v)m(ailable)i="" ([cf.)d(bishop,)i(1995].)0="" fh(3.5)56="" b(complexit)n(y)16="" b(con)n(trol)0="" fr(in)21="" b(previous)h(sections)f(w)o(e)g(ha)o(v)o="" (e)f(in)o(tro)q(duced)i(a)e(v)m(ariet)o(y)h(of)g(mo)q(dels)g(for)g="" (represen)o(ting)g(probabilit)o(y)h(distri-)0="" y(butions,)e(w)o(e)e="" (ha)o(v)o(e)g(sho)o(wn)g(ho)o(w)g(the)g(parameters)g(of)g(the)h(mo)q="" (dels)g(can)g(b)q(e)g(optimized)h(b)o(y)e(maximizing)i(the)0="" y(lik)o(eliho)q(o)q(d)k(function,)e(and)g(w)o(e)e(ha)o(v)o(e)h="" (outlined)h(a)f(n)o(um)o(b)q(er)g(of)f(p)q(o)o(w)o(erful)h(algorithms)g="" (for)f(p)q(erforming)i(this)0="" y(minimization.)j(before)17="" b(w)o(e)f(can)g(apply)h(this)g(framew)o(ork)e(in)i(practice)g(there)g="" (is)g(one)f(more)g(issue)h(w)o(e)f(need)i(to)0="" y(address,)d(whic)o="" (h)i(is)f(that)f(of)h(mo)q(del)g(complexit)o(y)l(.)23="" b(consider)16="" b(the)g(case)f(of)h(a)f(mixture)h(mo)q(del)h(giv)o(en)f="" (b)o(y)f(eq.)h(2.)0="" y(the)i(n)o(um)o(b)q(er)h(of)f(input)h(v)m="" (ariables)h(will)g(b)q(e)e(determined)i(b)o(y)e(the)g(particular)h="" (problem)g(at)f(hand.)29="" b(ho)o(w)o(ev)o(er,)0="" b(n)o(um)o(b)q(er)g="" fr(of)14="" b(comp)q(onen)o(t)h(densities)h="" (has)f(y)o(et)f(to)g(b)q(e)h(sp)q(eci\014ed.)22="" b(clearly)16="" b(if)f="" fr(is)15="" b(to)q(o)f(small)h(the)g(mo)q(del)0="" y(will)20="" b(b)q(e)e(insu\016cien)o(tly)j(\015exible)f(and)e(w)o(e)="" g(will)i(obtain)e(a)g(p)q(o)q(or)g(represen)o(tation)g(of)f(the)h(true)="" g(densit)o(y)l(.)30="" b(what)0="" y(is)19="" b(not)f(so)f(ob)o(vious)i(is)="" g(that)e(if)i="" fr(is)c(to)q(o)f(large)g(w)o(e)g(can)g(also)g="" (obtain)h(p)q(o)q(or)f(results.)30="" b(e\013ect)g(is)h(kno)o="" (wn)0="" y(as)e="" fq(over\014tting)g="" fr(and)h(arises)f(b)q(ecause)i(w)o="" (e)e(ha)o(v)o(e)g(a)g(data)g(set)g(of)g(\014nite)i(size.)28="" b(it)17="" b(is)h(illustrated)h(using)f(a)f(simple)0="" y(example)e(of)g(mixture)g(densit)o(y)g(estimation)g(in)g(figure)g(4.)k="" (here)c(a)g(set)f(of)g(100)g(data)g(p)q(oin)o(ts)h(in)g(one)g="" (dimension)0="" y(has)k(b)q(een)h(generated)f(from)g(a)f="" (distribution)j(consisting)f(of)f(a)f(mixture)i(of)f(t)o(w)o(o)e="" (gaussians)i(\(sho)o(wn)g(b)o(y)g(the)952="" y(14)p="" fm(i)16="" fi(th)68="" fr(comp)q(onen)o(t)18="" b(gaussian.)26="" b(a)17="" b(learning)i(algorithm)e(based)h(on)f(this)h="" (gradien)o(t)f(will)i(mo)o(v)o(e)e(the)g="" fm(i)1728="" fi(th)1780="" fr(mean)h="" fg(\026)1936="" fi(i)0="" fr(to)o(w)o(ard)c(the)h(data)f(p)q(oin)o(t)i="" fn(x)480="" fi(n)503="" fr(,)f(with)h(the)f(e\013ectiv)o(e)g(step)h(size)g="" (prop)q(ortional)f(to)g="" fm(h)1414="" fi(n;i)1459="" y(the)f(gradien)o(t)f(for)g(a)h(mixture)f(mo)q(del)i="" (will)g(alw)o(a)o(ys)e(tak)o(e)g(the)h(form)e(of)i(a)f(w)o(eigh)o(ted)h="" (sum)f(of)g(the)h(gradien)o(ts)0="" y(asso)q(ciated)f(with)h(the)f="" (comp)q(onen)o(t)g(mo)q(dels,)h(where)f(the)h(w)o(eigh)o(ts)e(are)h="" (the)g(p)q(osterior)g(probabilities)j(asso)q(ciated)0="" y(with)g(eac)o(h)f(of)g(the)h(comp)q(onen)o(ts.)k(the)c(k)o(ey)f="" (computational)h(issue)g(is)g(whether)g(these)f(p)q(osterior)h(w)o="" (eigh)o(ts)f(can)0="" y(b)q(e)h(computed)g(e\016cien)o(tly)l(.)22="" b(gaussian)g(mixture)h(mo)q(dels,)g(the)g(calculation)h="" (\(eq.)e(24\))f(is)i(clearly)h(e\016cien)o(t.)0="" y(f)l(or)d="" (decision)i(trees)d(there)i(are)f(a)f(set)h(of)g(p)q(osterior)g(w)o="" (eigh)o(ts)g(asso)q(ciated)h(with)f(eac)o(h)g(of)g(the)g(no)q(des)h(in)="" g(the)f(tree,)0="" y(and)i(a)f(recursion)i(is)f(a)o(v)m(ailable)h="" (that)f(computes)f(the)h(p)q(osterior)g(probabilities)i(in)e(an)g(up)o="" (w)o(ard)f(sw)o(eep)h([jordan)0="" y(and)i(jacobs,)g(1994].)27="" b(mixture)18="" b(mo)q(dels)h(in)f(the)g(form)f(of)h(a)f(c)o(hain)i(are)f="" (kno)o(wn)f(as)h(hidden)h(mark)o(o)o(v)d(mo)q(dels,)0="" y(and)i(the)h(calculation)h(of)d(the)i(relev)m(an)o(t)f(p)q="" (osterior)h(probabilities)h(is)f(p)q(erformed)f(via)h(an)f(e\016cien)o="" (t)h(algorithm)0="" y(kno)o(wn)c(as)g(the)g(baum-w)l(elc)o(h)h="" (algorithm.)91="" y(f)l(or)j(general)h(la)o(y)o(ered)f(net)o(w)o(ork)g="" (structures,)h(a)f(generic)h(algorithm)g(kno)o(wn)f(as)g(\\bac)o="" (kpropagation")f(is)0="" y(a)o(v)m(ailable)e(to)e(calculate)h(gradien)="" o(t)f(v)o(ectors)g([rumelhart,)g(et)h(al.,)f(1986].)k(bac)o="" (kpropagation)13="" b(is)i(essen)o(tially)h(the)0="" y(c)o(hain)g(rule)f="" (of)g(calculus)h(realized)h(as)d(a)h(graphical)g(algorithm.)20="" b(as)15="" b(applied)i(to)d(la)o(y)o(ered)h(net)o(w)o(orks)e(it)i(pro)o="" (vides)0="" y(a)f(simple)j(and)d(e\016cien)o(t)i(metho)q(d)f(that)f="" (calculates)i(a)e(gradien)o(t)h(in)g="" fm(w)6="" b(time)h(p)q(er)g(training)h(pattern,)e(where)0="" fm(w)21="" fr(is)16="" b(the)f(n)o(um)o(b)q(er)h(of)f(w)o(eigh)o="" (ts.)0="" fh(3.3)56="" b(optimization)16="" b(algorithms)0="" fr(by)d(in)o(tro)q(ducing)i(the)e(principle)j(of)d(maxim)o(um)g="" (lik)o(eliho)q(o)q(d)j(in)e(section)g(1,)f(w)o(e)g(ha)o(v)o(e)f="" (expressed)i(the)g(problem)f(of)0="" y(learning)i(in)f(neural)h(net)o="" (w)o(orks)d(in)i(terms)f(of)g(the)h(minimization)i(of)d(a)g(cost)g="" (function)i="" fm(j)t="" fr(\))d(whic)o(h)j(dep)q(ends)g(on)0="" y(a)e(v)o(ector)f="" fr(of)f(adaptiv)o(e)h(parameters.)19="" b(an)13="" b(imp)q(ortan)o(t)f(asp)q(ect)h(of)g(this)g(problem)g(is)h="" (that)e(the)h(gradien)o(t)g(v)o(ector)0="" fk(r)38="" fe(w)72="" fr(can)14="" b(b)q(e)i(ev)m(aluated)f="" (e\016cien)o(tly)h(\(for)e(example)i(b)o(y)e(bac)o(kpropagation\).)19="" b(gradien)o(t-based)c(minimization)0="" y(is)d(a)g(standard)f="" (problem)i(in)g(unconstrained)g(nonlinear)g(optimization,)g(for)e(whic)="" o(h)i(man)o(y)f(p)q(o)o(w)o(erful)g(tec)o(hniques)0="" y(ha)o(v)o(e)i(b)q(een)i(dev)o(elop)q(ed)g(o)o(v)o(er)d(the)i(y)o="" (ears.)k(suc)o(h)c(algorithms)f(generally)h(start)f(b)o(y)g(making)g="" (an)h(initial)h(guess)f(for)0="" y(the)g(parameter)g(v)o(ector)f="" fn(w)i="" fr(and)f(then)h(iterativ)o(ely)g(up)q(dating)g(the)f(v)o(ector)g="" (in)h(a)f(sequence)h(of)f(steps:)744="" fn(w)783="" fj(\()p="" fi(\034)t="" fj(+1\))889="" fn(w)976="" fj(\))1035="" fr(+)d(\001)p="" fn(w)1157="" fj(\))1869="" fr(\(25\))0="" y(where)j="" fm(\034)18="" fr(denotes)13="" b(the)f(step)h(n)o(um)o(b)q(er.)20="" b(initial)j(parameter)d(v)o="" (ector)g="" fn(w)1288="" fj(\(0\))1348="" fr(is)h(often)f(c)o="" (hosen)h(at)f(random,)h(and)0="" y(the)i(\014nal)i(v)o(ector)d="" (represen)o(ts)i(a)f(minim)o(um)h(of)f(the)h(cost)e(function)j(at)d="" (whic)o(h)j(the)e(gradien)o(t)g(v)m(anishes.)22="" b(due)16="" b(to)0="" y(the)f(nonlinear)h(nature)f(of)g(neural)h(net)o(w)o(ork)d="" (mo)q(dels,)j(the)f(cost)f(function)i(is)g(generally)g(a)e(highly)j="" (complicated)0="" y(function)d(of)g(the)f(parameters,)g(and)h(ma)o(y)="" e(p)q(ossess)i(man)o(y)f(suc)o(h)h(minima.)20="" b(di\013eren)o(t)14="" b(algorithms)f(di\013er)h(in)g(ho)o(w)0="" y(the)h(up)q(date)h(\001)p="" fn(w)308="" fj(\))372="" fr(is)f(computed.)91="" y(the)f(simplest)i(suc)o(h)e(algorithm)h(is)f(called)i="" fq(gr)n(adient)f(desc)n(ent)f="" fr(and)g(in)o(v)o(olv)o(es)h(a)f="" (parameter)f(up)q(date)i(whic)o(h)0="" y(is)f(prop)q(ortional)g(to)e="" (the)i(negativ)o(e)f(of)g(the)g(cost)g(function)h(gradien)o(t)g(\001)e="" (=")h" fk(\000)p="" fm(\021)r="" fk(r)p="" fm(e)i="" fr(where)f="" fm(\021)g="" fr(is)g(a)f(\014xed)h(constan)o(t)0="" y(called)21="" b(the)f(learning)h(rate.)33="" b(it)20="" b(should)h(b)q(e)g(stressed)e(that)="" g(gradien)o(t)h(descen)o(t)g(is)h(a)e(particularly)i(ine\016cien)o(t)0="" y(optimization)i(algorithm.)40="" b(v)l(arious)23="" b(mo)q="" (di\014cations)g(ha)o(v)o(e)e(b)q(een)i(prop)q(osed,)h(suc)o(h)f(as)e="" (the)h(inclusion)j(of)c(a)0="" fq(momentum)d="" fr(term,)g(to)f(try)g="" (to)g(impro)o(v)o(e)h(its)g(p)q(erformance.)27="" b(fact)e(m)o(uc)o(h)g(more)h(p)q(o)o(w)o(erful)g(algorithms)f(are)0="" y(readily)k(a)o(v)m(ailable,)i(as)c(describ)q(ed)j(in)f(standard)e="" (textb)q(o)q(oks)h(suc)o(h)g(as)g([fletc)o(her,)g(1987].)33="" b(tw)o(o)19="" b(of)h(the)g(b)q(est)0="" y(kno)o(wn)h(are)g(called)i="" fq(c)n(onjugate)f(gr)n(adients)f="" fq(quasi-newton)f="" fr(\(or)g="" fq(variable)h(metric)p="" fr(\))f(metho)q(ds.)39="" b(f)l(or)21="" b(the)0="" y(particular)16="" b(case)g(of)f(a)g="" (sum-of-squares)g(cost)g(function,)h(the)f="" fq(l)n(evenb)n(er)n(g{mar)n="" (quar)n(dt)31="" fr(algorithm)15="" b(can)h(also)f(b)q(e)0="" y(v)o(ery)g(e\013ectiv)o(e.)20="" b(soft)o(w)o(are)14="" b(implemen)o(tations)i(of)f(these)g(algorithms)h(are)e(widely)j(a)o(v)m="" (ailable.)952="" y(13)p="" fh(3.2)56="" b(gradien)n(ts)18="" b(of)h(the)f(cost)h="" fr(once)f(w)o(e)g(ha)o(v)o(e)f(de\014ned)i(a)e="" (probabilistic)j(mo)q(del,)e(obtained)g(a)g(cost)f(function)h(and)g="" (found)f(an)h(e\016cien)o(t)g(pro-)0="" y(cedure)23="" b(for)e(calculating)i(the)f(gradien)o(t)g(of)g(the)g(cost)f(function,)j="" (the)e(problem)g(can)g(b)q(e)h(handed)g(o\013)e(to)g(an)0="" y(optimization)g(routine.)35="" b(before)20="" b(discussing)h="" (optimization)g(pro)q(cedures,)h(ho)o(w)o(ev)o(er,)e(it)g(is)h(useful)g="" (to)e(exam-)0="" y(ine)f(the)e(form)g(that)g(the)h(gradien)o(t)g(tak)o="" (es)f(for)g(the)g(examples)i(that)e(w)o(e)g(ha)o(v)o(e)g(discussed)j="" (in)e(the)g(previous)g(t)o(w)o(o)0="" y(sections.)91="" y(the)e="" fm(i)200="" fi(th)249="" fr(output)g(unit)g(in)h(a)e="" (la)o(y)o(ered)h(net)o(w)o(ork)f(is)h(endo)o(w)o(ed)g(with)g(a)g(rule)g="" (for)g(com)o(bining)h(the)e(activ)m(ations)0="" y(of)h(units)h(in)h="" (earlier)f(la)o(y)o(ers,)f(yielding)j(a)d(quan)o(tit)o(y)h(that)f(w)o="" (e)g(denote)h(b)o(y)f="" fm(z)1289="" fi(i)1303="" fr(,)h(and)f(a)h(function)g(that)f(con)o(v)o(erts)g="" fm(z)1936="" fr(in)o(to)h(the)g(output)g="" fm(y)344="" fi(i)358="" b(regression)h(problems,)h(w)o="" (e)e(assume)h(linear)h(output)f(units)h(suc)o(h)f(that)f="" fm(y)1736="" fi(i)1764="" fm(z)1834="" fi(i)1849="" y(binary)17="" b(classi\014cation)g(problems,)g(our)f(earlier)h(discussion)h(sho)o(w)o="" (ed)e(that)f(a)h(natural)g(output)g(function)h(is)g(the)0="" y(logistic:)j="" fm(y)193="" fi(i)220="" b(1)p="" fr(\(1)8="" b(+)g="" fm(e)427="" fi(z)470="" fd(i)485="" fr(\).)19="" b(m)o(ulti-w)o(a)o="" (y)h(classi\014cation,)h(it)f(is)h(p)q(ossible)g(to)f(generalize)h(the)="" f(deriv)m(ation)h(of)0="" y(the)e(logistic)i(function)f(to)e(obtain)h="" (an)h(analogous)e(represen)o(tation)i(for)e(the)h(m)o(ulti-w)o(a)o(y)h="" (p)q(osterior)f(probabilities)0="" y(kno)o(wn)i(as)g(the)g="" fq(softmax)i(function)d="" fr([cf.)h(bishop,)h(1995]:)849="" fm(y)871="" fi(i)898="" fm(e)1012="" fi(z)1028="" fd(i)p="" ff(p)995="" fi(k)1024="" fm(e)1045="" fi(z)1061="" fd(k)1088="" fm(;)768="" fr(\(21\))0="" fm(y)153="" fi(i)183="" fr(represen)o(ts)g="" (the)g(p)q(osterior)h(probabilit)o(y)g(of)f(category)f="" fm(i)p="" y(if)g(w)o(e)f(no)o(w)g(consider)i(the)e(gradien)o="" (t)h(of)f="" fr(\))g(with)h(resp)q(ect)g(to)e="" fm(z)1219="" fi(i)1234="" fr(,)h(it)h(turns)f(out)h(that)e(w)o="" (e)i(obtain)f(a)h(single)0="" y(canonical)i(expression)h(of)d(the)i="" (follo)o(wing)g(form:)764="" fm(@)s(j)p="" a(@)s="" fn(w)842="" ff(x)914="" fi(i)950="" fm(t)984="" fi(i)1008="" fk(\000)11="" fm(y)1076="" fi(i)1090="" fr(\))1115="" fm(@)s(z)1163="" fn(w)1183="" fm(:)673="" fr(\(22\))0="" y(as)17="" b(discussed)i(b)o(y)e([rumelhart,)g(et)g="" (al.)26="" b(1995],)16="" b(this)i(form)e(for)g(the)i(gradien)o(t)f(is)g="" (predicted)i(from)d(the)h(theory)0="" y(of)j(generalized)h(linear)g="" (mo)q(dels)g([mccullagh)f(and)g(nelder,)i(1983],)d(where)i(it)f(is)g="" (sho)o(wn)g(that)f(the)h(linear,)0="" y(logistic,)15="" b(and)f(softmax)e(functions)j(are)e(\(in)o(v)o(erse\))h="" fq(c)n(anonic)n(al)f(links)j="" fr(for)e(the)f(gaussian,)h(bernoulli,)i="" (and)e(m)o(ulti-)0="" y(nomial)j(distributions,)i(resp)q(ectiv)o(ely)="" l(.)26="" b(canonical)17="" b(links)h(can)f(b)q(e)h(found)f(for)f(all)h(of)f="" (the)h(distributions)h(in)g(the)0="" y(exp)q(onen)o(tial)e(family)l="" (,)e(th)o(us)g(pro)o(viding)h(a)f(solid)i(statistical)e(foundation)h="" (for)e(handling)j(a)e(wide)h(v)m(ariet)o(y)f(of)g(data)0="" y(formats)g(at)g(the)i(output)f(la)o(y)o(er)g(of)f(a)h(net)o(w)o="" (ork,)f(including)k(coun)o(ts,)c(time)i(in)o(terv)m(als)g(and)g(rates.)="" y(the)e(gradien)o(t)f(of)h(the)f(cost)h(function)g(for)f="" (mixture)h(mo)q(dels)h(has)e(an)h(in)o(teresting)g(in)o(terpretation.)="" b(t)l(aking)0="" b(partial)h(deriv)m(ativ)o(e)g(of)f="" fr(\))g(in)h(eq.)e(20)h(with)h(resp)q(ect)f(to)g="" fg(\026)1147="" fi(i)1161="" fr(,)g(w)o(e)g(\014nd:)718="" fg(\026)769="" fi(i)800="" ff(x)868="" fi(n)916="" fm(h)942="" fi(n;i)987="" fg(\006)1030="" fi(i)1044="" fn(x)1090="" fi(n)1123="" fk(\000)c="" fg(\026)1201="" fi(i)1215="" fm(;)623="" fr(\(23\))0="" fm(h)157="" fi(n;i)218="" fr(is)h(de\014ned)g(as)f(follo)o(ws:)405="" fm(h)431="" fi(n;i)489="" fm(\031)619="" fi(i)633="" fg(\006)688="" fi(i)702="" fk(j)715="" fj(1)p="" fj(2)805="" fr(exp)p="" fk(f\000)937="" a(2)960="" fn(x)1006="" fi(n)1039="" fk(\000)10="" fg(\026)1117="" fi(i)1131="" fr(\))1149="" fi(t)1176="" fg(\006)1219="" fj(1)1219="" fi(i)1266="" fn(x)1312="" fi(n)1345="" fk(\000)g="" fg(\026)1422="" fi(i)1436="" fk(g)p="" ff(p)586="" fi(k)615="" fm(\031)641="" fi(k)662="" fg(\006)717="" fi(k)739="" fk(j)752="" fj(2)841="" fr(exp)q="" fk(f\000)974="" a(2)996="" fn(x)1042="" fi(n)1075="" fk(\000)h="" fg(\026)1153="" fi(k)1174="" fr(\))1192="" fi(t)1219="" fg(\006)1262="" fj(1)1262="" fi(k)1309="" fn(x)1355="" fi(n)1388="" fg(\026)1466="" fi(k)1487="" fk(g)1532="" fm(:)324="" fr(\(24\))0="" y(when)15="" b(summed)g(o)o(v)o(er)e="" fr(,)h(the)g(quan)o(tit)o(y)g="" fm(h)734="" fi(n;i)794="" fr(sums)h(to)e(one,)i(and)f(is)h(often)f(view)o(ed)i="" (as)e(the)g(\\resp)q(onsibilit)o(y")i(or)0="" y(\\credit")i(assigned)="" g(to)f(the)h="" fm(i)514="" fi(th)566="" fr(comp)q(onen)o(t)g(for)="" f(the)h="" fm(n)981="" fi(th)1034="" fr(data)f(p)q(oin)o(t.)27="" b(indeed,)20="" b(in)o(terpreting)e(eq.)f(24)g(using)0="" y(ba)o(y)o(es)e(rule)i(sho)o(ws)e(that)g="" fm(h)478="" fi(n;i)539="" fr(is)i(the)f(p)q(osterior)f(probabilit)o(y)i(that)="" f(the)f="" fm(n)1294="" fi(th)1346="" fr(data)g(p)q(oin)o(t)h(is)g="" (generated)g(b)o(y)g(the)952="" y(12)p="" fh(3.1)56="" b(lik)n(eliho)r(o)r(d-based)16="" b(cost)i(functions)0="" fr(regression,)13="" b(classi\014cation)g(and)g="" (densit)o(y)f(estimation)g(mak)o(e)g(di\013eren)o(t)g(probabilistic)i="" (assumptions)e(ab)q(out)g(the)0="" y(form)i(of)h(the)h(data)e(and)h="" (therefore)g(require)h(di\013eren)o(t)g(cost)e(functions.)91="" y(eq.)i(3)g(de\014nes)i(a)e(probabilistic)i(mo)q(del)g(for)e="" (regression.)24="" b(mo)q(del)i(is)f(a)f(conditional)i(densit)o="" (y)f(for)f(the)0="" y(targets)h="" fn(t)h="" fr(in)h(whic)o(h)g(the)g="" (targets)e(are)h(distributed)h(as)f(gaussian)g(random)g(v)m(ariables)i="" (\(assuming)e(gaussian)0="" y(errors)e="" fm(\017)p="" fr(\))g(with)h(mean)g="" (v)m(alues)g="" fm(f)5="" fr(\).)23="" b(no)o(w)f(write)g(the)h(conditional)h(mean)f(as)f="" fm(;)j="" b(to)i(mak)o(e)g(explicit)0="" y(the)21="" b(dep)q(endence)i(on)d(the)h(parameters)f="" fr(.)35="" b(giv)o(en)21="" b(the)g(training)g(set)g="" fk(d)i="" fk(f)p="" fn(x)1454="" fi(n)1476="" fn(t)1517="" fi(n)1540="" fk(g)1563="" fi(n)1563="" y(n)p="" fj(="1)1632" fr(,)21="" b(and)g(giv)o(en)g(our)0="" y(assumption)13="" b(that)f(the)h(targets)f="" fn(t)580="" fi(n)617="" fr(are)g(sampled)i(indep)q(enden)o(tly)i(\(giv)o="" (en)d(the)g(inputs)h="" fn(x)1532="" fi(n)1568="" fr(and)f(the)g(parameters)0="" fr(\),)h(w)o(e)h(obtain:)664="" fr(\))d(=")833" y(1)p="" a(2)869="" ff(x)888="" fi(n)936="" fk(k)p="" fn(t)979="" fi(n)1013="" fk(\000)e="" fn(x)1131="" fi(n)1154="" fk(k)1255="" fj(2)1273="" fm(;)583="" fr(\(17\))0="" y(where)13="" b(w)o(e)g(ha)o(v)o(e)g(assumed)g(an)g="" (iden)o(tit)o(y)h(co)o(v)m(ariance)g(matrix)e(and)i(dropp)q(ed)g(those)="" f(terms)f(that)h(do)g(not)f(dep)q(end)0="" y(on)18="" b(the)f="" (parameters.)26="" b(cost)g(function)g(is)g(the)g(standard)f="" (least)g(squares)h(cost)f(function)h(whic)o(h)h(is)f(tradi-)0="" y(tionally)e(used)g(in)g(neural)f(net)o(w)o(ork)f(training)i(for)e="" (real-v)m(alued)j(targets.)i(minimization)e(of)d(this)i(cost)e="" y(is)h(t)o(ypically)g(ac)o(hiev)o(ed)g(via)g(some)f="" (form)f(of)h(gradien)o(t)g(optimization,)h(as)f(w)o(e)f(discuss)j(in)f="" (the)f(follo)o(wing)h(section.)91="" y(classi\014cation)d(problems)f="" (di\013er)g(from)f(regression)h(problems)g(in)h(the)e(use)h(of)g="" (discrete-v)m(alued)i(targets,)d(and)0="" y(the)k(lik)o(eliho)q(o)q="" (d)i(accordingly)f(tak)o(es)e(a)g(di\013eren)o(t)h(form.)19="" b(binary)h(classi\014cation)h(the)f(bernoulli)i(probabilit)="" o(y)0="" y(mo)q(del)h="" fm(t)p="" fm(y)396="" fi(t)411="" fr(\(1)c="" fm(y)r="" fr(\))550="" fi(t)625="" b(natural,)g(where)g(w)o(e)g(use)g="" fm(y)i="" fr(to)d(denote)h(the)g(probabilit)o(y)h="" fm(t)d="" y(this)16="" b(mo)q(del)g(yields)h(the)e(follo)o(wing)h(log)="" f(lik)o(eliho)q(o)q(d:)539="" fk(\000)746="" ff(x)765="" fi(n)813="" fr([)p="" fm(t)842="" fi(n)873="" fr(ln)c="" fm(y)941="" fi(n)975="" fr(+)h(\(1)g="" fm(t)1132="" fi(n)1156="" fr(\))e(ln\(1)i="" fm(y)1337="" fi(n)1361="" fr(\)])d="" fm(;)457="" fr(\(18\))0="" y(whic)o(h)19="" b(is)f(kno)o(wn)g(as)f(the)h="" fq(cr)n(oss)g(entr)n(opy)k="" fr(function.)28="" b(it)18="" b(can)g(b)q(e)g="" (minimized)j(using)d(the)g(same)g(generic)h(opti-)0="" y(mization)d(pro)q(cedures)g(as)f(are)g(used)g(for)g(least)g(squares.)="" y(f)l(or)22="" b(m)o(ulti-w)o(a)o(y)g(classi\014cation)i(problems)="" f(in)g(whic)o(h)h(there)e(are)g="" fm(c)j="" fr(categories,)f(where)f="" fm(c)k(="">)e Fr(2,)f(the)0 1639 y(m)o(ultinomial)17 b(distribution)f(is)g (natural.)k(De\014ne)c Fn(t)885 1646 y Fi(n)923 1639 y Fr(suc)o(h)g(that)e(its)h(elemen)o(ts)h Fm(t)1389 1646 y Fi(n;i)1450 1639 y Fr(are)f(one)g(or)g(zero)f(according)0 1695 y(to)20 b(whether)i(the)f Fm(n)353 1679 y Fi(th)409 1695 y Fr(data)f(p)q(oin)o(t)i(b)q(elongs)g(to)e(the)h Fm(i)973 1679 y Fi(th)1029 1695 y Fr(category)l(,)g(and)h(de\014ne)g Fm(y)1482 1702 y Fi(n;i)1548 1695 y Fr(to)f(b)q(e)g(the)g(net)o(w)o (ork's)0 1752 y(estimate)c(of)h(the)f(p)q(osterior)h(probabilit)o(y)h (of)e(category)g Fm(i)g Fr(for)g(data)g(p)q(oin)o(t)h Fm(n)p Fr(;)h(i.e.,)e Fm(y)1489 1759 y Fi(n;i)1552 1752 y Fk(\021)g Fm(p)p Fr(\()p Fm(t)1661 1759 y Fi(n;i)1723 1752 y Fr(=)g(1)p Fk(j)p Fn(x)1839 1759 y Fi(n)1861 1752 y Fm(;)8 b Fn(w)q Fr(\).)0 1808 y(Giv)o(en)16 b(these)f(de\014nitions)i (w)o(e)e(obtain)g(the)h(follo)o(wing)g(cost)e(function:)707 1902 y Fm(J)t Fr(\()p Fn(w)q Fr(\))e(=)h Fk(\000)914 1861 y Ff(X)933 1948 y Fi(n)981 1861 y Ff(X)1005 1952 y Fi(i)1049 1902 y Fm(t)1065 1909 y Fi(n;i)1118 1902 y Fr(ln)8 b Fm(y)1185 1909 y Fi(n;i)1231 1902 y Fm(;)625 b Fr(\(19\))0 2029 y(whic)o(h)16 b(again)f(has)g(the)h(form)e(of)h(a)g (cross)f(en)o(trop)o(y)l(.)91 2086 y(W)l(e)i(no)o(w)g(turn)g(to)f (densit)o(y)i(estimation)f(as)g(exempli\014ed)j(b)o(y)d(Gaussian)g (mixture)g(mo)q(deling.)24 b(The)17 b(prob-)0 2142 y(abilistic)j(mo)q (del)f(in)f(this)h(case)e(is)i(that)e(giv)o(en)h(in)g(Eq.)g(2.)27 b(Assuming)18 b(Gaussian)g(comp)q(onen)o(t)g(densities)i(with)0 2198 y(arbitrary)15 b(co)o(v)m(ariance)g(matrices,)g(w)o(e)g(obtain)h (the)f(follo)o(wing)h(cost)e(function:)319 2310 y Fm(J)t Fr(\()p Fn(w)q Fr(\))f(=)f Fk(\000)526 2269 y Ff(X)546 2356 y Fi(n)594 2310 y Fr(ln)639 2269 y Ff(X)663 2360 y Fi(i)707 2310 y Fm(\031)733 2317 y Fi(i)809 2279 y Fr(1)p 752 2299 138 2 v 752 2342 a Fk(j)p Fg(\006)807 2349 y Fi(i)821 2342 y Fk(j)834 2329 y Fj(1)p Fi(=)p Fj(2)902 2310 y Fr(exp)979 2250 y Ff(\032)1010 2310 y Fk(\000)1050 2279 y Fr(1)p 1050 2299 23 2 v 1050 2341 a(2)1078 2310 y(\()p Fn(x)1124 2317 y Fi(n)1157 2310 y Fk(\000)e Fg(\026)1234 2320 y Fi(i)1248 2310 y Fr(\))1266 2291 y Fi(T)1293 2310 y Fg(\006)1336 2290 y Fl(\000)p Fj(1)1336 2323 y Fi(i)1383 2310 y Fr(\()p Fn(x)1429 2317 y Fi(n)1462 2310 y Fk(\000)h Fg(\026)1540 2320 y Fi(i)1554 2310 y Fr(\))1572 2250 y Ff(\033)1610 2310 y Fm(;)246 b Fr(\(20\))0 2437 y(where)19 b(the)g(parameters)f Fn(w)h Fr(are)g(the)f(collection)j(of)d(mean)h(v)o(ectors)f Fg(\026)1250 2448 y Fi(i)1264 2437 y Fr(,)i(the)f(co)o(v)m(ariance)g (matrices)g Fg(\006)1830 2444 y Fi(i)1845 2437 y Fr(,)g(and)0 2494 y(the)f(mixing)g(prop)q(ortions)g Fm(\031)506 2501 y Fi(i)520 2494 y Fr(.)27 b(A)18 b(similar)h(cost)e(function)h(arises)g (for)f(the)h(generalized)h(mixture)f(mo)q(dels)h(\(cf.)0 2550 y(Eq.)c(12\).)952 2775 y(11)p eop %%Page: 10 20 10 19 bop 0 192 a Fp(3)69 b(Learning)23 b(from)g(Data)0 293 y Fr(The)16 b(previous)h(section)g(has)f(pro)o(vided)h(a)e (selection)j(of)e(mo)q(dels)h(to)e(c)o(ho)q(ose)h(from;)g(w)o(e)f(no)o (w)h(face)g(the)g(problem)0 349 y(of)h(matc)o(hing)g(these)h(mo)q(dels) g(to)f(data.)26 b(In)18 b(principle)i(the)d(problem)h(is)g(straigh)o (tforw)o(ard:)k(giv)o(en)c(a)f(family)h(of)0 406 y(mo)q(dels)f(of)e(in) o(terest)h(w)o(e)f(attempt)g(to)g(\014nd)i(out)e(ho)o(w)h(probable)g (eac)o(h)g(of)g(these)g(mo)q(dels)g(is)h(in)f(the)g(ligh)o(t)g(of)g (the)0 462 y(data.)i(W)l(e)13 b(can)g(then)f(select)i(the)e(most)g (probable)h(mo)q(del)h(\(a)d(selection)j(rule)g(kno)o(wn)e(as)g Fq(maximum)j(a)f(p)n(osteriori)0 519 y Fr(or)k Fq(MAP)k Fr(estimation\),)c(or)g(w)o(e)f(can)i(select)g(some)e(highly)j (probable)f(subset)f(of)g(mo)q(dels,)h(w)o(eigh)o(ted)f(b)o(y)g(their)0 575 y(probabilit)o(y)g(\(an)e(approac)o(h)h(that)f(w)o(e)h(discuss)g(b) q(elo)o(w)h(in)f(the)g(section)h(on)e(Ba)o(y)o(esian)h(metho)q(ds\).)25 b(In)17 b(practice)0 632 y(there)g(are)f(a)g(n)o(um)o(b)q(er)h(of)f (problems)h(to)f(solv)o(e,)g(b)q(eginning)j(with)d(the)h(sp)q (eci\014cation)h(of)e(the)h(family)g(of)f(mo)q(dels)0 688 y(of)i(in)o(terest.)30 b(In)19 b(the)f(simplest)i(case,)f(in)g (whic)o(h)g(the)g(family)g(can)g(b)q(e)g(describ)q(ed)h(as)e(a)g (\014xed)h(structure)g(with)0 745 y(v)m(arying)c(parameters)e(\(e.g.,)g (the)h(class)h(of)e(feedforw)o(ard)h(MLP's)f(with)i(a)f(\014xed)g(n)o (um)o(b)q(er)h(of)f(hidden)i(units\),)e(the)0 801 y(learning)i(problem) g(is)g(essen)o(tially)g(one)g(of)e Fq(p)n(ar)n(ameter)j(estimation)p Fr(.)j(If)15 b(on)g(the)g(other)g(hand)h(the)f(family)h(is)f(not)0 858 y(easily)f(view)o(ed)g(as)f(a)g(\014xed)g(parametric)g(family)h (\(e.g.,)e(feedforw)o(ard)g(MLP's)h(with)h(v)m(ariable)g(n)o(um)o(b)q (er)f(of)g(hidden)0 914 y(units\),)i(then)h(w)o(e)f(m)o(ust)f(solv)o(e) i(the)f Fq(mo)n(del)h(sele)n(ction)h Fr(problem.)91 970 y(In)h(this)g(section)g(w)o(e)f(discuss)i(the)f(parameter)e(estimation) i(problem.)28 b(The)17 b(goal)h(will)h(b)q(e)f(to)f(\014nd)h(MAP)0 1027 y(estimates)13 b(of)f(the)i(parameters)e(b)o(y)h(maximizing)h(the) f(probabilit)o(y)i(of)d(the)h(parameters)f(giv)o(en)i(the)f(data)f Fk(D)q Fr(.)20 b(W)l(e)0 1083 y(compute)15 b(this)h(probabilit)o(y)g (using)g(Ba)o(y)o(es)f(rule:)736 1207 y Fm(p)p Fr(\()p Fn(w)q Fk(jD)q Fr(\))c(=)947 1176 y Fm(p)p Fr(\()p Fk(D)q(j)p Fn(w)q Fr(\))p Fm(p)p Fr(\()p Fn(w)q Fr(\))p 947 1196 243 2 v 1021 1238 a Fm(p)p Fr(\()p Fk(D)q Fr(\))1194 1207 y Fm(;)662 b Fr(\(13\))0 1331 y(where)15 b(w)o(e)g(see)g(that)f (to)g(calculate)i(MAP)f(estimates)g(w)o(e)f(m)o(ust)h(maximize)h(the)e (expression)i(in)g(the)f(n)o(umerator)0 1387 y(\(the)f(denominator)g (do)q(es)h(not)e(dep)q(end)j(on)e Fn(w)q Fr(\).)19 b(Equiv)m(alen)o (tly)d(w)o(e)e(can)g(minimize)j(the)d(negativ)o(e)g(logarithm)g(of)0 1444 y(the)h(n)o(umerator.)k(W)l(e)c(th)o(us)g(de\014ne)i(the)e(follo)o (wing)h Fq(c)n(ost)g(function)i Fm(J)t Fr(\()p Fn(w)q Fr(\):)671 1546 y Fm(J)t Fr(\()p Fn(w)q Fr(\))12 b(=)h Fk(\000)8 b Fr(ln)g Fm(p)p Fr(\()p Fk(D)q(j)p Fn(w)q Fr(\))h Fk(\000)h Fr(ln)f Fm(p)p Fr(\()p Fn(w)q Fr(\))p Fm(;)588 b Fr(\(14\))0 1648 y(whic)o(h)16 b(w)o(e)g(wish)g(to)e (minimize)k(with)e(resp)q(ect)g(to)f(the)g(parameters)g Fn(w)q Fr(.)20 b(The)c(\014rst)f(term)g(in)h(this)g(cost)f(function)0 1704 y(is)e(a)g(\(negativ)o(e\))g(log)g(lik)o(eliho)q(o)q(d.)22 b(If)13 b(w)o(e)g(assume)f(that)h(the)g(elemen)o(ts)g(in)h(the)f (training)h(set)f Fk(D)h Fr(are)f(conditionally)0 1761 y(indep)q(enden)o(t)j(of)d(eac)o(h)h(other)f(giv)o(en)i(the)e (parameters,)g(then)h(the)g(lik)o(eliho)q(o)q(d)i(factorizes)e(in)o(to) g(a)f(pro)q(duct)h(form.)0 1817 y(F)l(or)h(densit)o(y)g(estimation)h(w) o(e)f(ha)o(v)o(e:)755 1910 y Fm(p)p Fr(\()p Fk(D)q(j)p Fn(w)q Fr(\))c(=)978 1857 y Fi(N)968 1870 y Ff(Y)961 1960 y Fi(n)p Fj(=1)1035 1910 y Fm(p)p Fr(\()p Fn(x)1104 1917 y Fi(n)1127 1910 y Fk(j)p Fn(w)q Fr(\))672 b(\(15\))0 2027 y(and)15 b(for)g(classi\014cation)i(and)e(regression)g(w)o(e)g(ha) o(v)o(e:)716 2165 y Fm(p)p Fr(\()p Fk(D)q(j)p Fn(w)q Fr(\))d(=)940 2112 y Fi(N)929 2125 y Ff(Y)923 2216 y Fi(n)p Fj(=1)997 2165 y Fm(p)p Fr(\()p Fn(t)1058 2172 y Fi(n)1081 2165 y Fk(j)p Fn(x)1122 2172 y Fi(n)1145 2165 y Fm(;)c Fn(w)q Fr(\))p Fm(:)633 b Fr(\(16\))0 2301 y(In)13 b(b)q(oth)g(cases)f(this)h(yields)i(a)d(log)h(lik)o(eliho)q(o)q (d)i(whic)o(h)e(is)h(the)e(sum)h(of)f(the)h(log)f(probabilities)j(for)d (eac)o(h)h(individual)0 2358 y(data)i(p)q(oin)o(t.)21 b(F)l(or)15 b(the)g(remainder)h(of)f(this)h(section)g(w)o(e)f(will)i (assume)f(this)f(additiv)o(e)i(form;)d(moreo)o(v)o(er,)g(w)o(e)h(will)0 2414 y(assume)22 b(that)f(the)h(log)h(prior)f(probabilit)o(y)h(of)f (the)g(parameters)f(is)i(uniform)f(across)f(the)i(parameters)e(and)0 2470 y(drop)c(the)h(second)g(term.)26 b(Th)o(us)17 b(w)o(e)h(fo)q(cus)f (on)h Fq(maximum)h(likeliho)n(o)n(d)i Fr(\(ML\))c(estimation,)h(where)g (w)o(e)f(c)o(ho)q(ose)0 2527 y(parameter)d(v)m(alues)j Fn(w)391 2534 y Fi(M)t(L)469 2527 y Fr(that)e(maximize)h(ln)8 b Fm(p)p Fr(\()p Fk(D)q(j)p Fn(w)q Fr(\).)952 2775 y(10)p eop %%Page: 9 21 9 20 bop 0 192 a Fr(reject)22 b(noisy)h(input)g(v)m(ariables)g(that)f (carry)f(little)j(information)e(ab)q(out)g(the)g(output.)41 b(Ov)o(erlapping)24 b(basis)0 248 y(functions)16 b(are)f(often)g(view)o (ed)h(as)f(yielding)i(lo)o(w)o(er)e(v)m(ariance)h(predictions)h(and)e (as)g(b)q(eing)h(more)f(robust.)0 366 y Fh(2.5)56 b(General)17 b(mixture)f(mo)r(dels)0 452 y Fr(The)d(use)g(of)f(mixture)h(mo)q(dels)h (is)f(not)f(restricted)h(to)g(densit)o(y)g(estimation;)g(rather,)g(the) f(mixture)h(approac)o(h)g(can)0 508 y(b)q(e)j(used)g(quite)h(generally) f(to)f(build)j(complex)e(mo)q(dels)h(out)e(of)g(simple)i(parts.)k(T)l (o)15 b(illustrate,)h(let)g(us)g(consider)0 565 y(using)f(mixture)g(mo) q(dels)g(to)f(mo)q(del)h(a)f(conditional)i(densit)o(y)f(in)h(the)e(con) o(text)g(of)g(a)g(regression)h(or)e(classi\014cation)0 621 y(problem.)22 b(A)16 b(mixture)g(mo)q(del)g(in)h(this)f(setting)g (is)g(referred)g(to)f(as)g(a)g(\\mixtures)h(of)f(exp)q(erts")h(mo)q (del)h([Jacobs,)0 678 y(et)e(al.,)g(1991].)91 734 y(Supp)q(ose)k(that)e (w)o(e)h(ha)o(v)o(e)f(at)h(our)f(disp)q(osal)j(an)e(elemen)o(tal)h (conditional)g(mo)q(del)g Fm(p)p Fr(\()p Fn(t)p Fk(j)p Fn(x)p Fm(;)8 b Fn(w)q Fr(\).)26 b(Consider)18 b(a)0 791 y(situation)f(in)g(whic)o(h)g(the)g(conditional)h(mean)e(or)g (discriminan)o(t)i(exhibits)g(v)m(ariation)f(on)f(a)g(lo)q(cal)i(scale) f(that)e(is)0 847 y(a)k(go)q(o)q(d)g(matc)o(h)g(to)f(our)h(elemen)o (tal)h(mo)q(del,)h(but)e(the)h(v)m(ariation)g(di\013ers)f(in)h (di\013eren)o(t)f(regions)h(of)f(the)g(input)0 904 y(space.)i(W)l(e)16 b(could)g(use)g(a)f(more)g(complex)i(net)o(w)o(ork)d(to)h(try)g(to)g (capture)g(this)h(global)g(v)m(ariation;)g(alternativ)o(ely)0 960 y(w)o(e)j(migh)o(t)f(wish)i(to)e(com)o(bine)h(lo)q(cal)h(v)m(arian) o(ts)f(of)f(our)h(elemen)o(tal)h(mo)q(dels)f(in)h(some)e(manner.)31 b(This)19 b(can)g(b)q(e)0 1017 y(ac)o(hiev)o(ed)d(b)o(y)f(de\014ning)i (the)e(follo)o(wing)h(probabilistic)i(mixture:)621 1133 y Fm(p)p Fr(\()p Fn(t)p Fk(j)p Fn(x)p Fm(;)8 b Fn(w)q Fr(\))i(=)871 1080 y Fi(M)859 1092 y Ff(X)861 1184 y Fi(i)p Fj(=1)927 1133 y Fm(p)p Fr(\()p Fm(i)p Fk(j)p Fn(x)p Fm(;)e Fn(v)q Fr(\))o Fm(p)p Fr(\()p Fn(t)p Fk(j)p Fn(x)p Fm(;)f(i)o(;)h Fn(w)1287 1140 y Fi(i)1298 1133 y Fr(\))p Fm(:)540 b Fr(\(11\))0 1249 y(Comparing)20 b(this)h(mixture)f(to)g(the)g(unconditional)i(mixture)f(de\014ned)g (earlier)g(\(Eq.)f(2\),)g(w)o(e)g(see)g(that)g(b)q(oth)0 1306 y(the)e(mixing)h(prop)q(ortions)g(and)f(the)g(comp)q(onen)o(t)g (densities)i(are)e(no)o(w)g(conditional)i(densities)f(dep)q(enden)o(t)h (on)0 1362 y(the)f(input)h(v)o(ector)e Fn(x)p Fr(.)30 b(The)19 b(former)f(dep)q(endence)j(is)f(particularly)g(imp)q(ortan)o (t|w)o(e)e(no)o(w)h(view)g(the)g(mixing)0 1419 y(prop)q(ortion)j Fm(p)p Fr(\()p Fm(i)p Fk(j)p Fn(x)p Fm(;)8 b Fn(v)q Fr(\))19 b(as)j(pro)o(viding)h(a)f(probabilistic)i(device)g(for)e(c)o(ho)q (osing)g(di\013eren)o(t)h(elemen)o(tal)g(mo)q(dels)0 1475 y(\(\\exp)q(erts"\))12 b(in)j(di\013eren)o(t)e(regions)h(of)e(the) i(input)g(space.)20 b(A)13 b(learning)i(algorithm)e(that)f(c)o(ho)q (oses)i(v)m(alues)g(for)f(the)0 1532 y(parameters)g Fn(v)i Fr(as)e(w)o(ell)i(as)f(the)g(v)m(alues)h(for)f(the)g(parameters)f Fn(w)1106 1539 y Fi(i)1134 1532 y Fr(can)h(b)q(e)h(view)o(ed)g(as)f (attempting)g(to)f(\014nd)i(b)q(oth)0 1588 y(a)g(go)q(o)q(d)g (partition)h(of)e(the)i(input)g(space)f(and)h(a)f(go)q(o)q(d)g(\014t)g (to)f(the)i(lo)q(cal)g(mo)q(dels)g(within)g(that)f(partition.)91 1645 y(This)23 b(approac)o(h)e(can)h(b)q(e)h(extended)g(recursiv)o(ely) g(b)o(y)f(considering)i(mixtures)e(of)g(mo)q(dels)h(where)f(eac)o(h)0 1701 y(mo)q(del)14 b(ma)o(y)f(itself)i(b)q(e)f(a)f(mixture)h(mo)q(del)h ([Jordan)e(and)h(Jacobs,)f(1994].)18 b(Suc)o(h)c(a)g(recursion)g(can)g (b)q(e)g(view)o(ed)g(as)0 1758 y(pro)o(viding)h(a)f(probabilistic)j(in) o(terpretation)d(for)g(the)g(decision)i(trees)e(discussed)i(in)f(the)g (previous)g(section.)20 b(W)l(e)0 1814 y(view)d(the)f(decisions)i(in)f (the)f(decision)i(tree)e(as)g(forming)g(a)f(recursiv)o(e)i(set)f(of)g (probabilistic)i(selections)g(among)0 1870 y(a)f(set)h(of)f(mo)q(dels.) 28 b(The)18 b(total)f(probabilit)o(y)h(of)g(a)f(target)f Fn(t)i Fr(giv)o(en)g(an)g(input)g Fn(x)f Fr(is)h(the)g(sum)g(across)f (all)h(paths)0 1927 y(do)o(wn)d(the)g(tree:)353 2034 y Fm(p)p Fr(\()p Fn(t)p Fk(j)p Fn(x)p Fm(;)8 b Fn(w)q Fr(\))i(=)603 1981 y Fi(M)591 1994 y Ff(X)593 2085 y Fi(i)p Fj(=1)659 2034 y Fm(p)p Fr(\()p Fm(i)p Fk(j)p Fn(x)p Fm(;)e Fn(u)p Fr(\))842 1981 y Fi(M)831 1994 y Ff(X)830 2085 y Fi(j)r Fj(=1)899 2034 y Fm(p)p Fr(\()p Fm(j)s Fk(j)p Fn(x)p Fm(;)g(i;)f Fn(v)1087 2041 y Fi(i)1099 2034 y Fr(\))h Fk(\001)g(\001)g(\001)t Fm(p)p Fr(\()p Fn(t)p Fk(j)p Fn(x)p Fm(;)g(i;)g(j;)g(:)f(:)g(:)t(;)h Fn(w)1500 2041 y Fi(ij)r Fl(\001\001\001)1559 2034 y Fr(\))p Fm(;)279 b Fr(\(12\))0 2155 y(where)18 b Fm(i)g Fr(and)g Fm(j)j Fr(are)c(the)i(decisions)g(made)f(at)g(the)g(\014rst)g (lev)o(el)h(and)f(second)h(lev)o(el)g(of)f(the)g(tree,)g(resp)q(ectiv)o (ely)l(,)0 2211 y(and)f Fm(p)p Fr(\()p Fn(t)p Fk(j)p Fn(x)p Fm(;)8 b(i;)g(j;)f(:)h(:)f(:)t(;)h Fn(w)406 2218 y Fi(ij)r(:::)465 2211 y Fr(\))16 b(is)i(the)f(elemen)o(tal)h(mo)q(del) g(at)e(the)i(leaf)f(of)g(the)g(tree)g(de\014ned)h(b)o(y)f(the)g (sequence)h(of)0 2268 y(decisions.)j(This)14 b(probabilistic)j(mo)q (del)d(is)g(a)g(conditional)h(hierarc)o(hical)h(mixture.)k(Finding)15 b(parameter)e(v)m(alues)0 2324 y Fn(u)p Fr(,)h Fn(v)84 2331 y Fi(i)98 2324 y Fr(,)g(etc.)20 b(to)14 b(\014t)h(this)g(mo)q(del) g(to)f(data)g(can)h(b)q(e)g(view)o(ed)h(as)e(\014nding)i(a)e(nested)h (set)g(of)f(partitions)h(of)f(the)h(input)0 2381 y(space)g(and)h (\014tting)f(a)g(set)g(of)g(lo)q(cal)h(mo)q(dels)g(within)h(the)e (partition.)91 2437 y(The)f(mixture)h(mo)q(del)g(approac)o(h)f(can)h(b) q(e)g(view)o(ed)g(as)f(a)g(sp)q(ecial)i(case)e(of)g(a)g(general)h (metho)q(dology)f(kno)o(wn)0 2494 y(as)19 b Fq(le)n(arning)g(by)h(c)n (ommitte)n(e)p Fr(.)33 b(Bishop)20 b([1995])e(pro)o(vides)i(a)f (discussion)i(of)e(committees;)i(w)o(e)e(will)j(also)d(meet)0 2550 y(them)c(in)h(the)f(section)h(on)f(Ba)o(y)o(esian)h(metho)q(ds)f (later)g(in)h(the)f(c)o(hapter.)964 2775 y(9)p eop %%Page: 8 22 8 21 bop 91 192 a Fr(The)14 b(second)h(common)e(net)o(w)o(ork)g(mo)q (del)i(is)f(obtained)h(b)o(y)f(c)o(ho)q(osing)g(the)g(basis)g (functions)h Fm(\036)1699 199 y Fi(j)1717 192 y Fr(\()p Fn(x)p Fr(\))e(in)i(Eq.)e(8)0 248 y(to)g(b)q(e)i(functions)g(of)f(the)g (radial)h(v)m(ariable)g Fn(x)7 b Fk(\000)i Fg(\026)846 259 y Fi(j)878 248 y Fr(where)14 b Fg(\026)1041 259 y Fi(j)1073 248 y Fr(is)g(the)h Fq(c)n(enter)e Fr(of)g(the)i Fm(j)s Fr(th)e(basis)h(function,)h(whic)o(h)0 304 y(giv)o(es)21 b(rise)g(to)f(the)h Fq(r)n(adial)h(b)n(asis)e(function)g Fr(\(RBF\))h(net)o(w)o(ork)e(mo)q(del.)38 b(The)21 b(most)f(common)g (example)i(uses)0 361 y(Gaussians)15 b(of)g(the)g(form:)549 426 y Fm(\036)576 433 y Fi(j)594 426 y Fr(\()p Fn(x)p Fr(\))d(=)h(exp)795 367 y Ff(\032)826 426 y Fk(\000)866 395 y Fr(1)p 866 416 23 2 v 866 457 a(2)894 426 y(\()p Fn(x)d Fk(\000)g Fg(\026)1027 437 y Fi(j)1045 426 y Fr(\))1063 407 y Fj(T)1091 426 y Fg(\006)1133 407 y Fl(\000)p Fj(1)1133 439 y Fi(j)1181 426 y Fr(\()p Fn(x)f Fk(\000)h Fg(\026)1314 437 y Fi(j)1332 426 y Fr(\))1350 367 y Ff(\033)1388 426 y Fm(:)468 b Fr(\(10\))0 527 y(Here)14 b(b)q(oth)g(the)h(mean)f(v)o (ector)f Fg(\026)577 538 y Fi(j)609 527 y Fr(and)i(the)f(co)o(v)m (ariance)g(matrix)g Fg(\006)1182 534 y Fi(j)1214 527 y Fr(are)g(considered)h(to)f(b)q(e)g(adaptiv)o(e)g(param-)0 584 y(eters.)23 b(The)16 b(curse)h(of)e(dimensionalit)o(y)k(is)d (alleviated)i(b)q(ecause)f(the)f(basis)h(functions)g(can)f(b)q(e)h(p)q (ositioned)h(and)0 640 y(orien)o(ted)g(in)h(input)g(space)f(suc)o(h)g (as)f(to)g(o)o(v)o(erla)o(y)g(the)h(regions)g(of)g(high)g(data)f (densit)o(y)i(and)f(hence)h(to)e(capture)0 697 y(the)j(nonlinear)i (correlations)f(b)q(et)o(w)o(een)f(input)i(v)m(ariables.)36 b(Indeed,)23 b(a)d(common)g(approac)o(h)f(to)h(training)h(an)0 753 y(RBF)c(net)o(w)o(ork)e(is)i(to)f(use)h(a)f(t)o(w)o(o-stage)f(pro)q (cedure)i([Bishop,)g(1995].)23 b(In)17 b(the)g(\014rst)f(stage)g(the)g (basis)h(function)0 810 y(parameters)e(are)h(determined)h(using)g(the)f (input)h(data)e(alone,)i(whic)o(h)g(corresp)q(onds)f(to)f(a)h(densit)o (y)h(estimation)0 866 y(problem)f(using)g(a)f(mixture)h(mo)q(del)h(in)f (whic)o(h)g(the)g(comp)q(onen)o(t)f(densities)i(are)e(giv)o(en)h(b)o(y) g(the)f(basis)h(functions)0 923 y Fm(\036)27 930 y Fi(j)45 923 y Fr(\()p Fn(x)p Fr(\).)24 b(In)17 b(the)g(second)h(stage)e(the)h (basis)g(function)h(parameters)e(are)g(frozen)h(and)g(the)g(second-la)o (y)o(er)g(w)o(eigh)o(ts)0 979 y Fm(w)33 986 y Fi(k)q(j)85 979 y Fr(are)e(found)h(b)o(y)f(standard)g(least-squares)g(optimization) h(pro)q(cedures.)0 1101 y Fh(2.4)56 b(Decision)17 b(trees)0 1187 y Fr(MLP)g(and)h(RBF)f(net)o(w)o(orks)f(are)h(often)g(con)o (trasted)g(in)h(terms)e(of)h(the)g(supp)q(ort)h(of)f(the)g(basis)h (functions)g(that)0 1243 y(comp)q(ose)i(them.)33 b(MLP)20 b(net)o(w)o(orks)e(are)i(often)f(referred)h(to)f(as)h(\\global,")g(giv) o(en)g(that)f(linear-logistic)k(basis)0 1299 y(functions)14 b(are)e(b)q(ounded)i(a)o(w)o(a)o(y)e(from)g(zero)g(o)o(v)o(er)g(a)h (signi\014can)o(t)g(fraction)g(of)f(the)h(input)h(space.)20 b(Accordingly)l(,)14 b(in)0 1356 y(an)f(MLP)l(,)h(eac)o(h)f(input)i(v)o (ector)e(generally)h(giv)o(es)g(rise)g(to)f(a)g(distributed)i(pattern)e (o)o(v)o(er)g(the)g(hidden)j(units.)k(RBF)0 1412 y(net)o(w)o(orks,)c (on)h(the)g(other)f(hand,)i(are)e(referred)h(to)g(as)f(\\lo)q(cal,")i (due)f(to)g(the)g(fact)f(that)g(their)i(Gaussian)f(basis)0 1469 y(functions)i(t)o(ypically)h(ha)o(v)o(e)f(supp)q(ort)f(o)o(v)o(er) g(a)h(lo)q(cal)g(region)g(of)g(the)f(input)i(space.)30 b(It)19 b(is)g(imp)q(ortan)o(t)g(to)f(note,)0 1525 y(ho)o(w)o(ev)o(er,) g(that)g(lo)q(cal)h(supp)q(ort)g(do)q(es)f(not)g(necessarily)i(mean)e (non-o)o(v)o(erlapping)i(supp)q(ort;)f(indeed,)i(there)d(is)0 1582 y(nothing)c(in)h(the)f(RBF)g(mo)q(del)h(that)e(prefers)h(basis)h (functions)f(that)f(ha)o(v)o(e)h(non-o)o(v)o(erlapping)h(supp)q(ort.)k (A)14 b(third)0 1638 y(class)h(of)g(mo)q(del)g(that)g(do)q(es)g(fo)q (cus)g(on)g(basis)g(functions)g(with)h(non-o)o(v)o(erlapping)f(supp)q (ort)g(is)h(the)f Fq(de)n(cision)g(tr)n(e)n(e)0 1695 y Fr(mo)q(del)k([Breiman,)f(et)g(al.,)g(1984].)26 b(A)18 b(decision)i(tree)e(is)g(a)g(regression)g(or)f(classi\014cation)i(mo)q (del)g(that)e(can)h(b)q(e)0 1751 y(view)o(ed)e(as)e(asking)h(a)g (sequence)h(of)e(questions)i(ab)q(out)e(the)h(input)h(v)o(ector.)j(Eac) o(h)c(question)g(is)h(implemen)o(ted)g(as)0 1808 y(a)f(linear)i (discriminan)o(t,)f(and)g(a)f(sequence)i(of)e(questions)h(can)f(b)q(e)h (view)o(ed)g(as)f(a)g(recursiv)o(e)i(partitioning)f(of)f(the)0 1864 y(input)h(space.)k(All)c(inputs)g(that)e(arriv)o(e)g(at)g(a)h (particular)g(leaf)h(of)e(the)h(tree)f(de\014ne)i(a)f(p)q(olyhedral)h (region)f(in)h(the)0 1920 y(input)i(space.)26 b(The)17 b(collection)i(of)e(suc)o(h)g(regions)g(can)g(b)q(e)h(view)o(ed)g(as)f (a)f(set)h(of)g(basis)g(functions.)26 b(Asso)q(ciated)0 1977 y(with)21 b(eac)o(h)f(basis)g(function)h(is)g(an)f(output)g(v)m (alue)i(whic)o(h)f(\(ideally\))g(is)g(close)g(to)e(the)i(a)o(v)o(erage) e(v)m(alue)i(of)f(the)0 2033 y(conditional)15 b(mean)e(\(for)f (regression\))h(or)g(discriminan)o(t)h(function)g(\(for)f (classi\014cation;)h(a)f(ma)s(jorit)o(y)f(v)o(ote)g(is)i(also)0 2090 y(used\).)27 b(Th)o(us)18 b(the)g(decision)h(tree)f(output)f(can)h (b)q(e)g(written)g(as)f(a)g(w)o(eigh)o(ted)h(sum)g(of)f(basis)h (functions)g(in)h(the)0 2146 y(same)c(manner)g(as)g(a)g(la)o(y)o(ered)g (net)o(w)o(ork.)91 2203 y(As)g(this)h(discussion)h(suggests,)e (decision)i(trees)e(and)h(MLP/RBF)g(neural)g(net)o(w)o(orks)e(are)h(b)q (est)h(view)o(ed)g(as)0 2259 y(b)q(eing)h(di\013eren)o(t)f(p)q(oin)o (ts)g(along)g(the)g(con)o(tin)o(uum)h(of)e(mo)q(dels)i(ha)o(ving)f(o)o (v)o(erlapping)g(or)g(non-o)o(v)o(erlapping)h(basis)0 2316 y(functions.)i(Indeed,)14 b(as)d(w)o(e)g(sho)o(w)g(in)i(the)e (follo)o(wing)i(section,)f(decision)h(trees)f(can)f(b)q(e)i(treated)e (probabilistically)0 2372 y(as)19 b(mixture)i(mo)q(dels,)g(and)f(in)h (the)f(mixture)g(approac)o(h)g(the)f(sharp)h(discriminan)o(t)i (function)e(b)q(oundaries)h(of)0 2429 y(classical)c(decision)g(trees)e (b)q(ecome)h(smo)q(othed,)e(yielding)k(partially-o)o(v)o(erlapping)f (basis)e(functions.)91 2485 y(There)20 b(are)f(tradeo\013s)g(asso)q (ciated)g(with)h(the)g(con)o(tin)o(uum)g(of)g(degree-of-o)o(v)o (erlap|in)h(particular,)g(non-)0 2541 y(o)o(v)o(erlapping)f(basis)g (functions)g(are)f(generally)h(view)o(ed)g(as)f(b)q(eing)i(easier)f(to) e(in)o(terpret,)j(and)e(b)q(etter)g(able)i(to)964 2775 y(8)p eop %%Page: 7 23 7 22 bop 266 150 a 22376157 17900920 2105016 27957248 27233648 48086466 startTexFig 266 150 a %%BeginDocument: ps/mlp.ps save userdict /IslandDrawDict 300 dict dup begin put /ncpoint errordict /nocurrentpoint get def errordict begin /nocurrentpoint { dup /pathbbox load eq { pop 0 0 1 1 } { ncpoint } ifelse } bind def end /image_raster { %% sw sh dw dh xs ys translate scale /sh exch def /sw exch def /imagebuf sw 7 add 8 idiv string def sw sh 1 [sw 0 0 sh 0 0] { currentfile imagebuf readhexstring pop } image } bind def /m {moveto} bind def /l {lineto} bind def /c {curveto} bind def /n {newpath} bind def /cl {closepath} bind def /ar { %% sa ea sx sy rot tx ty matrix currentmatrix 8 1 roll translate rotate scale n 0 0 1 5 3 roll arc setmatrix } bind def /arn { %% sa ea sx sy rot tx ty matrix currentmatrix 8 1 roll translate rotate scale n 0 0 1 5 3 roll arcn setmatrix } bind def /el { %% sx sy rot tx ty matrix currentmatrix 6 1 roll translate rotate scale n 0 0 1 0 360 arc setmatrix cl } bind def /bp {setlinejoin setlinewidth setrgbcolor} bind def /bpbw {setlinejoin setlinewidth setgray} bind def /lw {setlinewidth} bind def /lj {setlinejoin} bind def /gr {setgray} bind def /BPSIDE 32 def %% pixels per pattern side /PATFREQ 3.0 def %% pattern pixels per mm /dp_mat [PATFREQ 0 0 PATFREQ 0 0] def /dp_pw BPSIDE def %% pattern pixel width /dp_ph BPSIDE def %% pattern pixel height /dp_w dp_pw PATFREQ div def %% pattern mm width /dp_h dp_ph PATFREQ div def %% pattern mm height /dp_bs 1 def %% pattern bits per pixel /savemat matrix def /topmat matrix def /patmat matrix def /patpath { /inv exch def topmat setmatrix pathbbox %% get lo - hi indecies /hy exch dp_h div floor cvi def /hx exch dp_w div floor cvi def /ly exch dp_h div floor cvi def /lx exch dp_w div floor cvi def lx 1 hx { dp_w mul ly 1 hy { dp_h mul exch dup 3 1 roll exch patmat currentmatrix pop translate dp_pw dp_ph inv dp_mat dp_proc imagemask patmat setmatrix } for pop } for } bind def /setpattern { /blue exch def /green exch def /red exch def /freq exch def /bwidth exch def /bpside exch def /bstring exch def /onbits 0 def /offbits 0 def freq 0 {/y exch def /x exch def /xindex x 1 add 2 div bpside mul cvi def /yindex y 1 add 2 div bpside mul cvi def bstring yindex bwidth mul xindex 8 idiv add get not 1 7 xindex 8 mod sub bitshift and 0 ne {/onbits onbits 1 add def 1} {/offbits offbits 1 add def 0} ifelse } setscreen {} settransfer systemdict /setcmykcolor known { /fact 1 onbits offbits onbits add div sub def 1 red sub fact mul 1 green sub fact mul 1 blue sub fact mul 0 setcmykcolor } { offbits offbits onbits add div setgray } ifelse } bind def /dmatrix matrix def /dpi 72 0 dmatrix defaultmatrix dtransform dup mul exch dup mul add sqrt def /B {gsave bp stroke grestore} bind def %% brush: gr lw lj /Bbw {gsave bpbw stroke grestore} bind def %% brush: gr lw lj /F {gsave setrgbcolor eofill grestore} bind def %% fill: gr /Fbw {gsave setgray eofill grestore} bind def %% fill: gr /PB {gsave setlinejoin setlinewidth setpattern stroke grestore} bind def /PF {gsave eoclip patpath grestore} bind def /BB { gsave setrgbcolor setlinejoin setlinewidth strokepath clip patpath grestore } bind def /BLACK { 0.0 } bind def /CP {closepath} bind def /FI {eofill} bind def /E {exch} bind def /FF {findfont} bind def /GR {grestore} bind def /GS {gsave} bind def /MF {makefont} bind def /NP {newpath} bind def /RO {rotate} bind def /ST {stroke} bind def /SC {scale} bind def /SF {setfont} bind def /SG {setgray} bind def /SLC {setlinecap} bind def /SLJ {setlinejoin} bind def /SLW {setlinewidth} bind def /TR {translate} bind def /WHITE { 1.0 } bind def /m {moveto} bind def /r {rmoveto} bind def /l {lineto} bind def /sp {x 0 rmoveto} bind def /rl {rlineto} bind def /s {show} bind def /box { NP m l l l CP } bind def /pageboundary { NP m l l l CP } bind def /BS { % black stroke GS SLJ SLW BLACK SG ST GR } bind def /WS { % white stroke GS SLJ SLW WHITE SG ST GR } bind def /reencode_small_dict 12 dict def /ReencodeSmall { reencode_small_dict begin /new_codes_and_names E def /new_font_name E def /base_font_name E def /base_font_dict base_font_name FF def /newfont base_font_dict maxlength dict def base_font_dict { E dup /FID ne { dup /Encoding eq { E dup length array copy newfont 3 1 roll put } { E newfont 3 1 roll put } ifelse } { pop pop } ifelse } forall newfont /FontName new_font_name put new_codes_and_names aload pop new_codes_and_names length 2 idiv { newfont /Encoding get 3 1 roll put } repeat new_font_name newfont definefont pop end %reencode_small_dict } def /extended_Zapf [ 8#223 /a89 8#224 /a90 8#225 /a93 8#226 /a94 8#227 /a91 8#230 /a92 8#231 /a205 8#232 /a85 8#233 /a206 8#234 /a86 8#235 /a87 8#236 /a88 8#237 /a95 8#240 /a96 ] def /extended_Standard [ 29 /thorn 30 /yacute 31 /divide 128 /Acircumflex 129 /Adieresis 130 /Agrave 131 /Aring 132 /Atilde 133 /Ccedilla 134 /Eacute 135 /Ecircumflex 136 /Edieresis 137 /Egrave 138 /Iacute 139 /Icircumflex 140 /Idieresis 141 /Igrave 142 /Ntilde 143 /Oacute 144 /Ocircumflex 145 /Odieresis 146 /Ograve 147 /Otilde 148 /Scaron 149 /Uacute 150 /Ucircumflex 151 /Udieresis 152 /Ugrave 153 /Ydieresis 154 /Zcaron 155 /aacute 156 /acircumflex 157 /adieresis 158 /agrave 159 /aring 160 /atilde 161 /exclamdown 162 /cent 163 /sterling 164 /fraction 165 /yen 166 /florin 167 /section 168 /currency 169 /quotesingle 170 /quotedblleft 171 /guillemotleft 172 /guilsinglleft 173 /guilsinglright 174 /fi 175 /fl 176 /plusminus 177 /endash 178 /dagger 179 /daggerdbl 180 /periodcentered 181 /twosuperior 182 /paragraph 183 /bullet 184 /quotesinglebase 185 /quotedblbase 186 /quotedblright 187 /guillemotright 188 /ellipsis 189 /perthousand 190 /threesuperior 191 /questiondown 192 /mu 193 /grave 194 /acute 195 /circumflex 196 /tilde 197 /macron 198 /breve 199 /dotaccent 200 /dieresis 201 /onesuperior 202 /ring 203 /cedilla 204 /onequarter 205 /hungarumlaut 206 /ogonek 207 /caron 208 /emdash 209 /ccedilla 210 /copyright 211 /eacute 212 /ecircumflex 213 /edieresis 214 /egrave 215 /iacute 216 /icircumflex 217 /idieresis 218 /igrave 219 /logicalnot 220 /minus 221 /ntilde 222 /oacute 223 /ocircumflex 224 /odieresis 225 /AE 226 /onehalf 227 /ordfeminine 228 /ograve 229 /otilde 230 /registered 231 /scaron 232 /Lslash 233 /Oslash 234 /OE 235 /ordmasculine 236 /trademark 237 /uacute 238 /ucircumflex 239 /udieresis 240 /ugrave 241 /ae 242 /ydieresis 243 /zcaron 244 /Aacute 245 /dotlessi 246 /threequarters 247 /Eth 248 /lslash 249 /oslash 250 /oe 251 /germandbls 252 /multiply 253 /Yacute 254 /Thorn 255 /eth ] def /extended_Symbol [ ] def /extend_font { % stack: fontname newfontname E dup (ZapfDingbats) eq { cvn E cvn extended_Zapf ReencodeSmall } { dup (Symbol) eq { cvn E cvn extended_Symbol ReencodeSmall } { cvn E cvn extended_Standard ReencodeSmall } ifelse } ifelse } bind def /getfont { /f E def f cvn where { begin f cvn load exec SF end } { f 0 f length 8 sub getinterval dup dup length 1 add string /localfont exch def localfont exch 0 exch putinterval localfont dup length 1 sub (X) putinterval localfont extend_font localfont FF /xsz f f length 4 sub 4 getinterval cvi def /ysz f f length 8 sub 4 getinterval cvi def [ xsz 0 0 ysz neg 0 0 ] MF dup f cvn E def SF } ifelse } bind def /ul { % space drop thickness GS currentpoint currentlinewidth currentpoint NP m 6 -3 roll SLW 0 E r 0 rl ST SLW m GR } bind def /ss { currentpoint pop E m } bind def /image_raster { % sw sh dw dh xs ys TR SC /sh E def /sw E def /imagebuf sw 7 add 8 idiv string def sw sh 1 [sw 0 0 sh 0 0] { currentfile imagebuf readhexstring pop } image } bind def /imagemask_raster { TR SC /sh E def /sw E def /imagebuf sw 7 add 8 idiv string def sw sh false [sw 0 0 sh 0 0] { currentfile imagebuf readhexstring pop } imagemask } bind def /image_color_raster { % sw sh sd dw dh xs ys systemdict /colorimage known not { /colorimage /colimg load def } if TR SC /sd E def /sh E def /sw E def /imagebuf sw 3 mul sd mul 7 add 8 idiv string def sw sh sd [sw 0 0 sh 0 0] { currentfile imagebuf readhexstring pop } false 3 colorimage } bind def /nx { /x E def } bind def 0. nx gsave 2.83465 -2.83465 scale 0 -279.4 translate topmat currentmatrix pop n 28.417 76.083 m 102.67 43.694 l 1 Fbw n 102.67 43.694 m 100.29 45.83 l 99.488 43.98 l cl 0 Fbw n 28.417 76.083 m 99.891 44.905 l gsave 0 0.176 0 Bbw grestore n 27.194 76.694 m 41.556 47.056 l 1 Fbw n 41.556 47.056 m 41.144 50.221 l 39.327 49.34 l cl 0 Fbw n 27.194 76.694 m 40.235 49.781 l gsave 0 0.176 0 Bbw grestore n 56.222 76.694 m 104.19 46.444 l 1 Fbw n 104.19 46.444 m 102.17 48.913 l 101.09 47.206 l cl 0 Fbw n 56.222 76.694 m 101.63 48.06 l gsave 0 0.176 0 Bbw grestore n 119.78 75.472 m 48.278 45.833 l 1 Fbw n 48.278 45.833 m 51.461 46.06 l 50.688 47.925 l cl 0 Fbw n 119.78 75.472 m 51.075 46.993 l gsave 0 0.176 0 Bbw grestore n 29.028 110.92 m 53.472 80.667 l 1 Fbw n 53.472 80.667 m 52.354 83.656 l 50.784 82.387 l cl 0 Fbw n 29.028 110.92 m 51.569 83.022 l gsave 0 0.176 0 Bbw grestore n 28.722 111.53 m 115.5 77.611 l 1 Fbw n 115.5 77.611 m 113.05 79.653 l 112.31 77.773 l cl 0 Fbw n 28.722 111.53 m 112.68 78.713 l gsave 0 0.176 0 Bbw grestore n 120 76.611 m 109.61 47.583 l 1 Fbw n 109.61 47.583 m 111.93 50.532 l 109.69 51.331 l cl 0 Fbw n 120 76.611 m 110.81 50.931 l gsave 0 0.352 0 Bbw grestore n 118.78 110.92 m 61.333 79.444 l 1 Fbw n 61.333 79.444 m 65.021 80.114 l 63.882 82.193 l cl 0 Fbw n 118.78 110.92 m 64.452 81.153 l gsave 0 0.352 0 Bbw grestore n 56.361 76.833 m 118.08 76.833 l 1 Fbw gsave [2 1 2 1] 0 setdash 0 0.352 0 Bbw grestore n 58.667 110.92 m 116.11 79.444 l 1 Fbw n 116.11 79.444 m 113.56 82.193 l 112.42 80.114 l cl 0 Fbw n 58.667 110.92 m 112.99 81.153 l gsave 0 0.352 0 Bbw grestore n 57.667 112.06 m 119.39 112.06 l 1 Fbw gsave [2 1 2 1] 0 setdash 0 0.352 0 Bbw grestore n 56.528 76.389 m 46.139 47.361 l 1 Fbw n 46.139 47.361 m 48.453 50.31 l 46.221 51.109 l cl 0 Fbw n 56.528 76.389 m 47.337 50.709 l gsave 0 0.352 0 Bbw grestore n 120.55 112.75 m 120.55 81.889 l 1 Fbw n 120.55 81.889 m 121.74 85.445 l 119.37 85.445 l cl 0 Fbw n 120.55 112.75 m 120.55 85.445 l gsave 0 0.352 0 Bbw grestore 3.0094 3.0094 0 120.55 111.83 el 1 Fbw gsave 0 0.352 0 Bbw grestore 4.7851 4.7851 0 120.55 76.847 el 1 Fbw gsave 0 0.352 0 Bbw grestore n savemat currentmatrix pop [1 0 0 1 58.3611 125.278] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (1) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1.11396 55.9167 123.155] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (x) s savemat setmatrix 3.0094 3.0094 0 28.639 111.83 el 1 Fbw gsave 0 0.352 0 Bbw grestore 3.0094 3.0094 0 28.639 76.528 el 1 Fbw gsave 0 0.352 0 Bbw grestore n 56.915 112.75 m 56.915 81.889 l 1 Fbw n 56.915 81.889 m 58.1 85.445 l 55.729 85.445 l cl 0 Fbw n 56.915 112.75 m 56.915 85.445 l gsave 0 0.352 0 Bbw grestore 3.0094 3.0094 0 56.915 111.83 el 1 Fbw gsave 0 0.352 0 Bbw grestore 4.7851 4.7851 90 56.915 76.847 el 1 Fbw gsave 0 0.352 0 Bbw grestore n 43.75 42.833 m 105.47 42.833 l 1 Fbw gsave [2 1 2 1] 0 setdash 0 0.352 0 Bbw grestore 4.7851 4.7851 90 107.94 42.847 el 1 Fbw gsave 0 0.352 0 Bbw grestore 4.7851 4.7851 0 44.304 42.847 el 1 Fbw gsave 0 0.352 0 Bbw grestore n savemat currentmatrix pop [1 0 0 1 118.167 123.139] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic04000400) getfont () s 0.00 SG (x) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 121 124.972] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (d) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 131.083 82.1944] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (M) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 128.333 80.3611] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic04000400) getfont () s 0.00 SG (z) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 46.1389 81.8889] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (1) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 43.6944 80.0556] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic04000400) getfont () s 0.00 SG (z) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 44.6111 34.5278] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (1) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 42.1667 32.0833] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic04000400) getfont () s 0.00 SG (y) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 109.389 33.9167] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (c) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 106.639 32.0833] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic04000400) getfont () s 0.00 SG (y) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 26.2778 86.1667] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic04000400) getfont () s 0.00 SG (z) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 29.0278 88] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (0) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 26.5833 123.139] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic04000400) getfont () s 0.00 SG (x) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 29.8611 124.889] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (0) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 65.6944 27.1944] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Helvetica03600360) getfont () s 0.00 SG (Output) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 68.1389 34.5278] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Helvetica03600360) getfont () s 0.00 SG (Units) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 126.194 63.5556] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Helvetica03600360) getfont () s 0.00 SG (Hidden) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 129.25 70.8889] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Helvetica03600360) getfont () s 0.00 SG (Units) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 80.9722 122.833] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Helvetica03600360) getfont () s 0.00 SG (Input) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 80.9722 130.167] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Helvetica03600360) getfont () s 0.00 SG (Units) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 11.3056 111.528] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Helvetica03600360) getfont () s 0.00 SG (bias) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 11.6111 73.3333] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Helvetica03600360) getfont () s 0.00 SG (bias) s savemat setmatrix userdict /#copies 1 put showpage grestore end restore %%EndDocument endTexFig 0 1382 a Fr(Figure)17 b(3:)22 b(An)17 b(example)h(of)e(a)g(feed-forw)o (ard)g(net)o(w)o(ork)g(ha)o(ving)h(t)o(w)o(o)e(la)o(y)o(ers)i(of)f (adaptiv)o(e)h(w)o(eigh)o(ts.)24 b(The)16 b(bias)0 1438 y(parameters)11 b(in)h(the)g(\014rst)f(la)o(y)o(er)g(are)g(sho)o(wn)g (as)h(w)o(eigh)o(ts)f(from)g(an)g(extra)g(input)h(ha)o(ving)g(a)f (\014xed)i(v)m(alue)f(of)f Fm(x)1834 1445 y Fj(0)1867 1438 y Fr(=)i(1.)0 1495 y(Similarly)l(,)k(the)d(bias)h(parameters)f(in) i(the)e(second)h(la)o(y)o(er)g(are)f(sho)o(wn)g(as)h(w)o(eigh)o(ts)f (from)g(an)g(extra)g(hidden)j(unit,)0 1551 y(with)f(activ)m(ation)f (again)h(\014xed)f(at)g Fm(z)622 1558 y Fj(0)654 1551 y Fr(=)e(1.)0 1675 y(the)j(t)o(w)o(o)f(most)g(common)g(examples.)23 b(The)17 b(\014rst)e(of)h(these)g(is)h(called)g(the)f Fq(multilayer)h(p)n(er)n(c)n(eptr)n(on)e Fr(\(MLP\))h(and)0 1732 y(is)i(obtained)h(b)o(y)f(c)o(ho)q(osing)g(the)g(basis)g (functions)h(to)e(b)q(e)i(giv)o(en)f(b)o(y)g(linear-logistic)i (functions)f(\(Eq.)e(6\).)27 b(This)0 1788 y(leads)16 b(to)e(a)h(m)o(ultiv)m(ariate)i(nonlinear)f(function)g(that)f(can)g(b)q (e)h(expressed)g(in)g(the)f(form:)557 1913 y Fm(y)579 1920 y Fi(k)600 1913 y Fr(\()p Fn(x)p Fr(\))d(=)736 1860 y Fi(M)724 1873 y Ff(X)724 1964 y Fi(j)r Fj(=1)793 1913 y Fm(w)826 1920 y Fi(k)q(j)863 1913 y Fm(g)894 1841 y Ff( )947 1860 y Fi(d)926 1873 y Ff(X)928 1964 y Fi(i)p Fj(=1)994 1913 y Fm(w)1027 1920 y Fi(j)r(i)1057 1913 y Fm(x)1083 1920 y Fi(i)1107 1913 y Fr(+)e Fm(w)1185 1920 y Fi(j)r Fj(0)1221 1841 y Ff(!)1264 1913 y Fr(+)g Fm(w)1342 1920 y Fi(k)q Fj(0)1381 1913 y Fm(:)498 b Fr(\(9\))0 2042 y(Here)13 b Fm(w)138 2049 y Fi(j)r Fj(0)185 2042 y Fr(and)g Fm(w)304 2049 y Fi(k)q Fj(0)354 2042 y Fr(are)f Fq(bias)g Fr(parameters,)g(and)g(the)h(basis)g(functions)g(are)f (called)h Fq(hidden)h(units)p Fr(.)19 b(The)12 b(function)0 2098 y Fm(g)r Fr(\()p Fk(\001)p Fr(\))k(is)h(the)h(logistic)g(sigmoid)g (function)g(of)f(Eq.)g(6.)26 b(This)18 b(can)f(also)g(b)q(e)h(represen) o(ted)g(as)f(a)g(net)o(w)o(ork)f(diagram)0 2155 y(as)i(in)i(Figure)e (3.)30 b(Suc)o(h)19 b(a)g(mo)q(del)g(is)g(able)h(to)e(tak)o(e)g(accoun) o(t)g(of)g(the)h(in)o(trinsic)h(dimensionalit)o(y)h(of)d(the)h(data)0 2211 y(b)q(ecause)f(the)f(\014rst-la)o(y)o(er)g(w)o(eigh)o(ts)f Fm(w)652 2218 y Fi(j)r(i)699 2211 y Fr(can)h(adapt)g(and)g(hence)h (orien)o(t)f(the)h(surfaces)e(along)i(whic)o(h)f(the)h(basis)0 2268 y(function)d(resp)q(onse)g(is)g(constan)o(t.)j(It)d(has)f(b)q(een) h(demonstrated)f(that)g(mo)q(dels)h(of)f(this)g(form)g(can)g(appro)o (ximate)0 2324 y(to)f(arbitrary)g(accuracy)g(an)o(y)h(con)o(tin)o(uous) g(function,)g(de\014ned)h(on)e(a)h(compact)f(domain,)h(pro)o(vided)g (the)g(n)o(um)o(b)q(er)0 2381 y Fm(M)23 b Fr(of)18 b(hidden)i(units)f (is)g(su\016cien)o(tly)g(large.)29 b(The)18 b(MLP)g(mo)q(del)i(can)e(b) q(e)h(extended)g(b)o(y)f(considering)i(sev)o(eral)0 2437 y(successiv)o(e)d(la)o(y)o(ers)f(of)g(w)o(eigh)o(ts.)22 b(Note)15 b(that)h(the)g(use)g(of)g(nonlinear)h(activ)m(ation)g (functions)g(is)f(crucial,)h(since)g(if)0 2494 y Fm(g)r Fr(\()p Fk(\001)p Fr(\))e(in)k(Eq.)d(9)h(w)o(ere)g(replaced)i(b)o(y)e (the)g(iden)o(tit)o(y)l(,)h(the)g(net)o(w)o(ork)e(w)o(ould)h(reduce)h (to)f(sev)o(eral)g(successiv)o(e)i(linear)0 2550 y(transformations)14 b(whic)o(h)i(w)o(ould)f(itself)i(b)q(e)e(linear.)964 2775 y(7)p eop %%Page: 6 24 6 23 bop 0 192 a Fr(c)o(haracterized)15 b(b)o(y)f(the)g (linear-logistic)i(form.)j(Nonetheless)c(it)f(is)h(still)g(useful)g(to) f(retain)g(the)g(logistic)h(function)0 248 y(and)i(fo)q(cus)g(on)g Fq(nonline)n(ar)j Fr(represen)o(tations)c(for)h(the)f(function)i Fm(z)r Fr(.)25 b(This)17 b(is)g(the)g(approac)o(h)g(tak)o(en)f(within)i (the)0 304 y(neural)e(net)o(w)o(ork)e(\014eld.)91 361 y(T)l(o)19 b(summarize,)h(w)o(e)f(ha)o(v)o(e)g(iden)o(ti\014ed)i(t)o(w) o(o)d(functions)i(that)f(are)g(imp)q(ortan)o(t)g(for)f(regression)i (and)f(clas-)0 417 y(si\014cation,)i(resp)q(ectiv)o(ely:)30 b(the)20 b(conditional)h(mean)e(and)h(the)f(discriminan)o(t)i (function.)34 b(These)20 b(are)f(the)g(t)o(w)o(o)0 474 y(functions)f(that)f(are)h(of)f(concern)h(for)f(simple)i(linear)g(mo)q (dels)g(and,)f(as)f(w)o(e)h(no)o(w)f(discuss,)i(for)e(more)g(complex)0 530 y(nonlinear)g(mo)q(dels)f(as)e(w)o(ell.)0 651 y Fh(2.3)56 b(Nonlinear)17 b(regression)h(and)h(nonlinear)f(classi\014cation)0 737 y Fr(The)d(linear)g(regression)f(and)h(linear)g(discriminan)o(t)h (functions)f(in)o(tro)q(duced)h(in)f(the)f(previous)h(section)g(ha)o(v) o(e)f(the)0 793 y(merit)j(of)f(simplicit)o(y)l(,)j(but)d(are)h(sev)o (erely)g(restricted)g(in)g(their)g(represen)o(tational)g(capabilities.) 27 b(A)16 b(con)o(v)o(enien)o(t)0 850 y(w)o(a)o(y)f(to)g(see)i(this)f (is)h(to)e(consider)i(the)f(geometrical)h(in)o(terpretation)f(of)f (these)i(mo)q(dels.)23 b(When)16 b(view)o(ed)h(in)g(the)0 906 y Fm(d)p Fr(-dimensional)g Fn(x)p Fr(-space,)e(the)g(linear)i (regression)e(function)h Fn(w)1098 889 y Fj(T)1125 906 y Fn(x)10 b Fr(+)g Fm(w)1241 913 y Fj(0)1276 906 y Fr(is)16 b(constan)o(t)e(on)h(h)o(yp)q(er-planes)i(whic)o(h)0 962 y(are)g(orthogonal)f(to)h(the)g(v)o(ector)g Fn(w)q Fr(.)25 b(F)l(or)17 b(man)o(y)g(practical)h(applications)g(w)o(e)f (need)h(to)f(consider)h(m)o(uc)o(h)f(more)0 1019 y(general)i(classes)g (of)f(function.)31 b(W)l(e)19 b(therefore)f(seek)h(represen)o(tations)f (for)g(nonlinear)i(mappings)f(whic)o(h)h(can)0 1075 y(appro)o(ximate)13 b(an)o(y)h(giv)o(en)g(mapping)h(to)e(arbitrary)g(accuracy)l(.)20 b(One)15 b(w)o(a)o(y)e(to)g(ac)o(hiev)o(e)h(this)h(is)f(to)f(transform) g(the)0 1132 y(original)h Fn(x)f Fr(using)g(a)g(set)g(of)f Fm(M)18 b Fr(nonlinear)c(functions)g Fm(\036)953 1139 y Fi(j)971 1132 y Fr(\()p Fn(x)p Fr(\))e(where)h Fm(j)i Fr(=)e(1)p Fm(;)8 b(:)g(:)g(:)d(;)j(M)d Fr(,)12 b(and)h(then)h(to)e (form)g(a)h(linear)0 1188 y(com)o(bination)j(of)f(these)g(functions,)h (so)e(that:)762 1283 y Fm(y)784 1290 y Fi(k)806 1283 y Fr(\()p Fn(x)p Fr(\))d(=)929 1243 y Ff(X)951 1334 y Fi(j)997 1283 y Fm(w)1030 1290 y Fi(k)q(j)1067 1283 y Fm(\036)1094 1290 y Fi(j)1112 1283 y Fr(\()p Fn(x)p Fr(\))p Fm(:)703 b Fr(\(8\))0 1421 y(F)l(or)18 b(a)g(su\016cien)o(tly)h(large)f (v)m(alue)i(of)e Fm(M)5 b Fr(,)18 b(and)h(for)e(a)h(suitable)i(c)o (hoice)f(of)f(the)g Fm(\036)1405 1428 y Fi(j)1424 1421 y Fr(\()p Fn(x)p Fr(\),)f(suc)o(h)i(a)f(mo)q(del)h(has)f(the)0 1477 y(desired)j(`univ)o(ersal)g(appro)o(ximation')f(prop)q(erties.)36 b(A)20 b(familiar)h(example,)h(for)e(the)g(case)g(of)g(1-dimensional)0 1534 y(input)d(spaces,)f(is)h(the)f(simple)h(p)q(olynomial,)h(for)d (whic)o(h)i(the)f Fm(\036)1099 1541 y Fi(j)1118 1534 y Fr(\()p Fn(x)p Fr(\))f(are)g(simply)j(successiv)o(e)f(p)q(o)o(w)o (ers)e(of)h Fm(x)g Fr(and)0 1590 y(the)h Fm(w)q Fr('s)f(are)h(the)g(p)q (olynomial)h(co)q(e\016cien)o(ts.)26 b(Mo)q(dels)18 b(of)e(the)h(form)g (in)g(Eq.)g(8)g(ha)o(v)o(e)f(the)h(prop)q(ert)o(y)g(that)f(they)0 1647 y(can)f(b)q(e)h(expressed)g(as)f(net)o(w)o(ork)f(diagrams)h(in)h (whic)o(h)g(there)f(is)h(a)f Fq(single)e Fr(la)o(y)o(er)i(of)g(adaptiv) o(e)h(w)o(eigh)o(ts.)91 1703 y(There)i(are)f(a)g(v)m(ariet)o(y)g(of)g (families)i(of)e(functions)h(in)h(one)e(dimension)i(that)e(can)h(appro) o(ximate)f(an)o(y)g(con-)0 1760 y(tin)o(uous)g(function)h(to)e (arbitrary)h(accuracy)l(.)25 b(There)17 b(is,)h(ho)o(w)o(ev)o(er,)e(an) h(imp)q(ortan)o(t)f(issue)i(whic)o(h)g(m)o(ust)e(b)q(e)i(ad-)0 1816 y(dressed,)e(called)i(the)e Fq(curse)h(of)g(dimensionality)p Fr(.)k(If,)16 b(for)g(example,)g(w)o(e)g(consider)h(an)f Fm(M)1560 1800 y Fj(th)1595 1816 y Fr(-order)g(p)q(olynomial)0 1873 y(then)i(the)g(n)o(um)o(b)q(er)h(of)e(indep)q(enden)o(t)k(co)q (e\016cien)o(ts)e(gro)o(ws)d(as)i Fm(d)1117 1856 y Fi(M)1174 1873 y Fr([Bishop,)h(1995].)27 b(F)l(or)17 b(a)h(t)o(ypical)h(medium-)0 1929 y(scale)f(application)i(with,)e(sa)o(y)l(,)f(30)h(inputs)g(a)g (fourth-order)f(p)q(olynomial)i(\(whic)o(h)f(is)h(still)g(quite)f (restricted)g(in)0 1985 y(its)f(represen)o(tational)h(capabilit)o(y\))g (w)o(ould)f(ha)o(v)o(e)g(o)o(v)o(er)f(46,000)f(adjustable)j (parameters.)24 b(As)17 b(w)o(e)g(shall)h(see)f(in)0 2042 y(Section)g(3.3)f(in)h(order)f(to)g(ac)o(hiev)o(e)h(go)q(o)q(d)f (generalization)i(it)f(is)g(imp)q(ortan)o(t)f(to)f(ha)o(v)o(e)i(more)e (data)h(p)q(oin)o(ts)h(than)0 2098 y(adaptiv)o(e)d(parameters)e(in)i (the)g(mo)q(del,)g(and)g(this)g(is)g(a)f(serious)h(problem)g(for)f (metho)q(ds)g(that)g(ha)o(v)o(e)g(a)g(p)q(o)o(w)o(er)g(la)o(w)0 2155 y(or)i(exp)q(onen)o(tial)h(gro)o(wth)e(in)i(the)g(n)o(um)o(b)q(er) f(of)g(parameters.)91 2211 y(A)g(solution)h(to)f(the)g(problem)h(lies)h (in)f(the)f(fact)g(that,)f(for)g(most)h(real-w)o(orld)h(data)e(sets,)h (there)g(are)g(strong)0 2268 y(\(often)i(nonlinear\))i(correlations)f (b)q(et)o(w)o(een)g(the)g(input)h(v)m(ariables)g(suc)o(h)f(that)f(the)g (data)h(do)q(es)g(not)f(uniformly)0 2324 y(\014ll)k(the)f(input)h (space)f(but)g(is)g(e\013ectiv)o(ely)h(con\014ned)g(to)e(a)h(sub-space) g(whose)g(dimensionalit)o(y)i(is)e(called)i(the)0 2381 y Fq(intrinsic)c(dimensionality)f Fr(of)h(the)g(data.)29 b(W)l(e)18 b(can)h(tak)o(e)e(adv)m(an)o(tage)h(of)g(this)h(phenomenon)g (b)o(y)f(considering)0 2437 y(again)c(a)f(mo)q(del)i(of)f(the)f(form)h (in)g(Eq.)g(8)f(but)h(in)h(whic)o(h)f(the)g(basis)h(functions)f Fm(\036)1364 2444 y Fi(j)1382 2437 y Fr(\()p Fn(x)p Fr(\))f(are)h Fq(adaptive)g Fr(so)g(that)f(they)0 2494 y(themselv)o(es)19 b(con)o(tain)g(w)o(eigh)o(t)f(parameters)g(whose)g(v)m(alues)i(can)e(b) q(e)i(adjusted)e(in)i(the)e(ligh)o(t)h(of)f(the)h(observ)o(ed)0 2550 y(data)12 b(set.)19 b(Di\013eren)o(t)12 b(mo)q(dels)i(result)f (from)f(di\013eren)o(t)h(c)o(hoices)h(for)e(the)h(basis)g(functions,)g (and)g(here)h(w)o(e)e(consider)964 2775 y(6)p eop %%Page: 5 25 5 24 bop 503 150 a 14917438 11933944 4078469 13222133 36114186 39074365 startTexFig 503 150 a %%BeginDocument: ps/posterior.eps /MathWorks 150 dict begin /bdef {bind def} bind def /ldef {load def} bind def /xdef {exch def} bdef /xstore {exch store} bdef /c /clip ldef /cc /concat ldef /cp /closepath ldef /gr /grestore ldef /gs /gsave ldef /mt /moveto ldef /np /newpath ldef /cm /currentmatrix ldef /sm /setmatrix ldef /rc {rectclip} bdef /rf {rectfill} bdef /rm /rmoveto ldef /rl /rlineto ldef /s /show ldef /sc {setcmykcolor} bdef /sr /setrgbcolor ldef /w /setlinewidth ldef /cap /setlinecap ldef /pgsv () def /bpage {/pgsv save def} bdef /epage {pgsv restore} bdef /bplot /gsave ldef /eplot {stroke grestore} bdef /portraitMode 0 def /landscapeMode 1 def /dpi2point 0 def /FontSize 0 def /FMS { /FontSize xstore %save size off stack findfont [FontSize 0 0 FontSize neg 0 0] makefont setfont }bdef /reencode { exch dup where {pop load} {pop StandardEncoding} ifelse exch dup 3 1 roll findfont dup length dict begin { 1 index /FID ne {def}{pop pop} ifelse } forall /Encoding exch def currentdict end definefont pop } bdef /isroman { findfont /CharStrings get /Agrave known } bdef /FMSR { 3 1 roll 1 index dup isroman {reencode} {pop pop} ifelse exch FMS } bdef /csm { 1 dpi2point div -1 dpi2point div scale neg translate landscapeMode eq {90 rotate} if } bdef /SO { [] 0 setdash } bdef /DO { [.5 dpi2point mul 4 dpi2point mul] 0 setdash } bdef /DA { [6 dpi2point mul] 0 setdash } bdef /DD { [.5 dpi2point mul 4 dpi2point mul 6 dpi2point mul 4 dpi2point mul] 0 setdash } bdef /L { lineto stroke } bdef /MP { 3 1 roll moveto 1 sub {rlineto} repeat } bdef /AP { {rlineto} repeat } bdef /PP { closepath fill } bdef /DP { closepath stroke } bdef /MR { 4 -2 roll moveto dup 0 exch rlineto exch 0 rlineto neg 0 exch rlineto closepath } bdef /FR { MR stroke } bdef /PR { MR fill } bdef /L1i { { currentfile picstr readhexstring pop } image } bdef /tMatrix matrix def /MakeOval { newpath tMatrix currentmatrix pop translate scale 0 0 1 0 360 arc tMatrix setmatrix } bdef /FO { MakeOval stroke } bdef /PO { MakeOval fill } bdef /PD { 2 copy moveto lineto stroke } bdef currentdict end def MathWorks begin 0 cap 0 setlinejoin end MathWorks begin bpage bplot /dpi2point 12 def portraitMode 0216 7344 csm 528 206 5844 4719 MR c np 76 dict begin %Colortable dictionary /c0 { 0 0 0 sr} bdef /c1 { 1 1 1 sr} bdef /c2 { 1 0 0 sr} bdef /c3 { 0 1 0 sr} bdef /c4 { 0 0 1 sr} bdef /c5 { 1 1 0 sr} bdef /c6 { 1 0 1 sr} bdef /c7 { 0 1 1 sr} bdef /Helvetica /ISOLatin1Encoding 240 FMSR c1 0 0 6914 5184 PR 24 w DO 16 w SO 24 w c0 898 4612 mt 6256 4612 L 898 388 mt 6256 388 L 898 4612 mt 898 388 L 6256 4612 mt 6256 388 L 898 4612 mt 898 4612 L 6256 4612 mt 6256 4612 L 898 4612 mt 6256 4612 L 898 4612 mt 898 388 L 898 4612 mt 898 4612 L 898 4612 mt 898 4558 L 898 388 mt 898 442 L 832 4870 mt (0) s 2238 4612 mt 2238 4558 L 2238 388 mt 2238 442 L 2172 4870 mt (1) s 3577 4612 mt 3577 4558 L 3577 388 mt 3577 442 L 3511 4870 mt (2) s 4917 4612 mt 4917 4558 L 4917 388 mt 4917 442 L 4851 4870 mt (3) s 6256 4612 mt 6256 4558 L 6256 388 mt 6256 442 L 6190 4870 mt (4) s 898 4612 mt 952 4612 L 6256 4612 mt 6202 4612 L 530 4701 mt (0.0) s 898 2500 mt 952 2500 L 6256 2500 mt 6202 2500 L 530 2589 mt (0.5) s 898 388 mt 952 388 L 6256 388 mt 6202 388 L 530 477 mt (1.0) s 898 4612 mt 6256 4612 L 898 388 mt 6256 388 L 898 4612 mt 898 388 L 6256 4612 mt 6256 388 L 898 388 mt 898 388 L 6256 388 mt 6256 388 L gs 898 388 5359 4225 MR c np DA 46 0 46 0 47 0 46 0 46 0 46 0 46 0 47 0 46 0 46 0 46 1 46 0 46 0 47 1 46 0 46 1 46 1 46 1 47 2 46 3 46 3 46 3 46 5 47 6 46 8 46 9 46 12 46 14 46 17 47 21 46 25 46 30 46 36 46 41 47 48 46 56 46 65 46 73 46 82 47 93 46 103 46 114 46 124 46 136 47 145 46 155 46 164 46 172 46 178 46 182 47 185 46 186 46 183 46 178 46 172 47 162 46 149 46 135 46 117 46 98 47 77 46 55 46 31 46 7 46 -16 47 -41 46 -64 46 -86 46 -106 46 -124 46 -141 47 -155 46 -166 46 -174 46 -181 46 -184 47 -186 46 -184 46 -181 46 -175 46 -169 47 -161 46 -151 46 -142 46 -131 46 -120 46 -110 47 -98 46 -89 46 -79 46 -69 46 -61 47 -53 46 -45 46 -39 46 -33 46 -28 47 -24 46 -19 1683 4532 100 MP stroke 46 -16 46 -13 46 -11 47 -8 46 -7 46 -6 46 -4 46 -4 46 -2 47 -3 46 -1 46 -1 46 -1 46 -1 47 -1 46 0 46 0 898 4611 18 MP stroke 46 0 46 0 47 1 46 1 46 1 46 1 46 1 47 3 46 2 46 4 46 4 46 6 46 7 47 8 46 11 46 13 46 16 46 19 47 24 46 28 46 33 46 39 46 45 47 53 46 61 46 69 46 79 46 89 46 98 47 110 46 120 46 131 46 142 46 151 47 161 46 169 46 175 46 181 46 184 47 186 46 184 46 181 46 174 46 166 47 155 46 141 46 124 46 106 46 86 46 64 47 41 46 16 46 -7 46 -31 46 -55 47 -77 46 -98 46 -117 46 -135 46 -149 47 -162 46 -172 46 -178 46 -183 46 -186 47 -185 46 -182 46 -178 46 -172 46 -164 46 -155 47 -145 46 -136 46 -124 46 -114 46 -103 47 -93 46 -82 46 -73 46 -65 46 -56 47 -48 46 -41 46 -36 46 -30 46 -25 46 -21 47 -17 46 -14 46 -12 46 -9 46 -8 47 -6 46 -5 46 -3 46 -3 46 -3 47 -2 46 -1 1683 4608 100 MP stroke 46 -1 46 -1 46 0 47 -1 46 0 46 0 46 -1 46 0 46 0 47 0 46 0 46 0 46 0 46 0 47 0 46 0 46 0 898 4612 18 MP stroke SO 46 4 46 5 47 6 46 5 46 6 46 7 46 7 47 7 46 8 46 9 46 9 46 10 46 10 47 11 46 12 46 13 46 14 46 15 47 15 46 17 46 18 46 18 46 21 47 21 46 23 46 24 46 26 46 27 46 29 47 30 46 32 46 35 46 36 46 37 47 40 46 42 46 44 46 46 46 49 47 50 46 52 46 55 46 57 46 58 47 61 46 63 46 65 46 66 46 69 46 69 47 72 46 72 46 74 46 74 46 76 47 76 46 76 46 77 46 77 46 76 47 76 46 76 46 74 46 74 46 72 47 72 46 69 46 69 46 66 46 65 46 63 47 61 46 58 46 57 46 55 46 52 47 50 46 49 46 46 46 44 46 42 47 40 46 37 46 36 46 35 46 32 46 30 47 29 46 27 46 26 46 24 46 23 47 21 46 21 46 18 46 18 46 17 47 15 46 15 1683 593 100 MP stroke 46 14 46 13 46 12 47 11 46 10 46 10 46 9 46 9 46 8 47 7 46 7 46 7 46 6 46 5 47 6 46 5 46 4 898 450 18 MP stroke 0 -4224 3577 4612 2 MP stroke gr end eplot epage end showpage %%EndDocument endTexFig 0 1004 a Fr(Figure)18 b(2:)26 b(This)19 b(sho)o(ws)e(the)h(Gaussian)h (class-conditional)h(densities)g Fm(p)p Fr(\()p Fm(x)p Fk(jC)1364 1011 y Fj(1)1383 1004 y Fr(\))d(\(dashed)i(curv)o(es\))f (for)f(a)h(t)o(w)o(o-)0 1060 y(class)12 b(problem)h(in)g(one)f (dimension,)i(together)d(with)h(the)g(corresp)q(onding)h(p)q(osterior)f (probabilit)o(y)h Fm(p)p Fr(\()p Fk(C)1754 1067 y Fj(1)1773 1060 y Fk(j)p Fm(x)p Fr(\))e(\(solid)0 1117 y(curv)o(e\))17 b(whic)o(h)i(tak)o(es)d(the)i(form)f(of)g(a)g(logistic)h(sigmoid.)28 b(The)18 b(v)o(ertical)g(line)h(sho)o(ws)e(the)g(decision)i(b)q (oundary)0 1173 y(for)c Fm(y)f Fr(=)f(0)p Fm(:)p Fr(5)h(whic)o(h)i (coincides)i(with)d(the)g(p)q(oin)o(t)h(at)f(whic)o(h)h(the)f(t)o(w)o (o)f(densit)o(y)i(curv)o(es)f(cross.)91 1307 y(It)d(is)h(easily)g(v)o (eri\014ed)h(that)d(if)i(the)g(class)f(conditional)i(densities)g(are)e (m)o(ultiv)m(ariate)i(Gaussians)e(with)g(iden)o(ti-)0 1363 y(cal)g(co)o(v)m(ariance)g(matrices,)f(then)h Fm(z)h Fr(is)f(a)f(linear)h(function)g(of)f Fn(x)p Fr(:)18 b Fm(z)c Fr(=)f Fn(w)1209 1347 y Fi(T)1236 1363 y Fn(x)r Fr(+)r Fm(w)1336 1370 y Fj(0)1356 1363 y Fr(.)18 b(Moreo)o(v)o(er)10 b(this)i(represen)o(tation)0 1420 y(is)19 b(appropriate)f(for)g(an)o(y) g(distribution)i(in)f(a)f(broad)g(class)h(of)f(densities)i(kno)o(wn)e (as)g(the)g(exp)q(onen)o(tial)i(family)0 1476 y(\(whic)o(h)e(includes)h (the)e(Gaussian,)h(the)f(P)o(oisson,)g(the)h(gamma,)e(the)h(binomial,)i (and)e(man)o(y)g(other)g(densities\).)0 1532 y(All)f(of)f(the)h (densities)g(in)g(this)g(family)g(can)f(b)q(e)h(put)f(in)i(the)e(follo) o(wing)h(form:)522 1635 y Fm(g)r Fr(\()p Fn(x)p Fr(;)8 b Fm(\022)q(;)g(\036)p Fr(\))i(=)j(exp)q Fk(f)p Fr(\()p Fm(\022)892 1616 y Fi(T)919 1635 y Fn(x)d Fk(\000)g Fm(b)p Fr(\()p Fm(\022)q Fr(\)\))p Fm(=a)p Fr(\()p Fm(\036)p Fr(\))f(+)h Fm(c)p Fr(\()p Fn(x)p Fm(;)e(\036)p Fr(\))p Fk(g)p Fm(;)462 b Fr(\(7\))0 1737 y(where)16 b Fm(\022)i Fr(is)f(the)f Fq(lo)n(c)n(ation)h(p)n(ar)n(ameter)p Fr(,)f(and)g Fm(\036)h Fr(is)f(the)h Fq(sc)n(ale)f(p)n(ar)n(ameter)p Fr(.)23 b(Substituting)17 b(this)g(general)g(form)e(in)0 1793 y(Eq.)j(5,)h(where)g Fm(\022)i Fr(is)e(allo)o(w)o(ed)g(to)f(v)m (ary)h(b)q(et)o(w)o(een)g(the)g(classes)g(and)g Fm(\036)g Fr(is)g(assumed)g(to)f(b)q(e)i(constan)o(t)e(b)q(et)o(w)o(een)0 1850 y(classes,)e(w)o(e)g(see)g(that)f Fm(z)j Fr(is)f(in)f(all)h(cases) f(a)g(linear)h(function.)23 b(Th)o(us)15 b(the)h(c)o(hoice)h(of)f(a)f (linear-logistic)k(mo)q(del)d(is)0 1906 y(rather)f(robust.)91 1962 y(The)21 b(geometry)e(of)h(the)h(t)o(w)o(o-class)e(problem)j(is)f (sho)o(wn)f(in)h(Figure)g(2,)g(whic)o(h)g(sho)o(ws)f(Gaussian)g(class-) 0 2019 y(conditional)d(densities,)f(and)g(suggests)e(the)i(logistic)g (form)e(of)h(the)g(p)q(osterior)g(probabilit)o(y)l(.)91 2075 y(The)g(function)h Fm(z)i Fr(in)e(our)e(analysis)i(is)g(an)f (example)h(of)f(a)g(discriminan)o(t)i(function.)j(In)c(general)g(a)f (discrim-)0 2132 y(inan)o(t)j(function)h(is)f(an)o(y)g(function)h(that) e(can)h(b)q(e)h(used)f(to)f(decide)j(on)e(class)g(mem)o(b)q(ership)h (\(Duda)f(and)g(Hart,)0 2188 y(1972\);)12 b(our)i(analysis)g(has)g(pro) q(duced)h(a)e(particular)h(form)f(of)h(discriminan)o(t)h(function)f (that)f(is)i(an)e(in)o(termediate)0 2245 y(step)k(in)g(the)g (calculation)h(of)f(a)f(p)q(osterior)h(probabilit)o(y)l(.)25 b(Note)17 b(that)f(if)h(w)o(e)f(set)h Fm(z)g Fr(=)f(0,)g(from)g(the)h (form)f(of)g(the)0 2301 y(logistic)21 b(function)f(w)o(e)g(obtain)g(a)f (probabilit)o(y)i(of)e(0.5,)h(whic)o(h)g(sho)o(ws)f(that)g Fm(z)k Fr(=)d(0)g(is)g(a)f Fq(de)n(cision)h(b)n(oundary)0 2358 y Fr(b)q(et)o(w)o(een)c(the)f(t)o(w)o(o)f(classes.)91 2414 y(The)k(discriminan)o(t)h(function)f(that)f(w)o(e)h(found)g(for)f (exp)q(onen)o(tial)i(family)f(densities)i(is)e(linear)h(under)f(the)0 2471 y(giv)o(en)k(conditions)h(on)e Fm(\036)p Fr(.)39 b(In)22 b(more)f(general)g(situations,)i(in)g(whic)o(h)f(the)g (class-conditional)h(densities)g(are)0 2527 y(more)17 b(complex)h(than)g(a)f(single)i(exp)q(onen)o(tial)f(family)h(densit)o (y)l(,)f(the)g(p)q(osterior)f(probabilit)o(y)i(will)g(not)e(b)q(e)h(w)o (ell)964 2775 y(5)p eop %%Page: 4 26 4 25 bop 0 192 a Fq(estimation)15 b Fr(and)g Fq(factor)i(analysis)d Fr([Anderson,)h(1984].)0 312 y Fh(2.2)56 b(Linear)17 b(regression)h(and)h(linear)f(discriminan)n(ts)0 398 y Fr(Regression)i(mo)q(dels)g(and)f(classi\014cation)h(mo)q(dels)g(b)q (oth)f(fo)q(cus)g(on)g(the)g(conditional)i(densit)o(y)f Fm(p)p Fr(\()p Fn(t)p Fk(j)p Fn(x)p Fr(\).)30 b(They)0 454 y(di\013er)19 b(in)h(that)e(in)h(regression)g(the)g(target)f(v)o (ector)g Fn(t)h Fr(is)g(a)g(real-v)m(alued)i(v)o(ector,)d(whereas)h(in) g(classi\014cation)i Fn(t)0 511 y Fr(tak)o(es)14 b(its)i(v)m(alues)g (from)f(a)f(discrete)j(set)d(represen)o(ting)i(the)g(class)f(lab)q (els.)91 567 y(The)j(simplest)g(probabilistic)i(mo)q(del)f(for)e (regression)g(is)i(one)e(in)i(whic)o(h)f Fn(t)g Fr(is)g(view)o(ed)g(as) g(the)f(sum)h(of)f(an)0 624 y(underlying)g(deterministic)g(function)f Fm(f)5 b Fr(\()p Fn(x)p Fr(\))15 b(and)g(a)g(Gaussian)g(random)g(v)m (ariable)h Fm(\017)p Fr(:)846 718 y Fn(t)d Fr(=)g Fm(f)5 b Fr(\()p Fn(x)p Fr(\))k(+)i Fm(\017:)788 b Fr(\(3\))0 813 y(If)17 b Fm(\017)f Fr(has)h(zero)f(mean,)g(as)g(is)h(commonly)g (assumed,)f Fm(f)5 b Fr(\()p Fn(x)p Fr(\))15 b(then)i(b)q(ecomes)g(the) g Fq(c)n(onditional)f(me)n(an)k Fm(E)s Fr(\()p Fn(t)p Fk(j)p Fn(x)p Fr(\).)h(It)0 870 y(is)f(this)g(function)h(that)e(is)h (the)f(fo)q(cus)h(of)f(most)g(regression)h(mo)q(deling.)35 b(Of)20 b(course,)g(the)g(conditional)h(mean)0 926 y(describ)q(es)14 b(only)g(the)e(\014rst)h(momen)o(t)f(of)g(the)h(conditional)h (distribution,)h(and,)e(as)f(w)o(e)g(discuss)i(in)g(a)e(later)h (section,)0 983 y(a)i(go)q(o)q(d)g(regression)g(mo)q(del)h(will)h(also) e(generally)i(rep)q(ort)e(information)g(ab)q(out)g(the)g(second)h (momen)o(t.)91 1039 y(In)g(a)f(linear)i(regression)f(mo)q(del)g(the)g (conditional)h(mean)e(is)h(a)g(linear)g(function)h(of)e Fn(x)p Fr(:)20 b Fm(E)s Fr(\()p Fn(t)p Fk(j)p Fn(x)p Fr(\))11 b(=)j Fm(W)6 b Fn(x)p Fr(,)15 b(for)0 1096 y(a)g(\014xed)h (matrix)f Fm(W)6 b Fr(.)20 b(Linear)d(regression)e(has)g(a)g(straigh)o (tforw)o(ard)f(represen)o(tation)h(as)g(a)g(net)o(w)o(ork)f(diagram)h (in)0 1152 y(whic)o(h)j(the)f Fm(j)234 1135 y Fi(th)286 1152 y Fr(input)h(unit)g(represen)o(ts)f(the)g Fm(j)825 1135 y Fi(th)877 1152 y Fr(comp)q(onen)o(t)g(of)g(the)g(input)h(v)o (ector)f Fm(x)1533 1159 y Fi(j)1551 1152 y Fr(,)g(eac)o(h)g(output)g (unit)h Fm(i)0 1208 y Fr(tak)o(es)e(the)g(w)o(eigh)o(ted)h(sum)g(of)f (the)h(input)g(v)m(alues,)h(and)f(the)f(w)o(eigh)o(t)h Fm(w)1239 1215 y Fi(ij)1285 1208 y Fr(is)g(placed)h(on)e(the)h(link)h (b)q(et)o(w)o(een)f(the)0 1265 y Fm(j)22 1248 y Fi(th)72 1265 y Fr(input)f(unit)g(and)f(the)g Fm(i)471 1248 y Fi(th)521 1265 y Fr(output)g(unit.)91 1321 y(The)f(conditional)h(mean)f (is)h(also)e(an)h(imp)q(ortan)o(t)g(function)g(in)h(classi\014cation)g (problems,)g(but)e(most)g(of)h(the)0 1378 y(fo)q(cus)i(in)g (classi\014cation)h(is)e(on)h(a)f(di\013eren)o(t)g(function)h(kno)o(wn) f(as)g(a)g Fq(discriminant)h(function)p Fr(.)k(T)l(o)15 b(see)h(ho)o(w)f(this)0 1434 y(function)h(arises)f(and)g(to)g(relate)g (it)h(to)e(the)h(conditional)i(mean,)e(w)o(e)g(consider)h(a)e(simple)j (t)o(w)o(o-class)d(problem)i(in)0 1491 y(whic)o(h)h(the)f(target)e(is)j (a)f(simple)h(binary)f(scalar)g(that)g(w)o(e)f(no)o(w)h(denote)g(b)o(y) g Fm(t)p Fr(.)22 b(The)16 b(conditional)i(mean)e Fm(E)s Fr(\()p Fm(t)p Fk(j)p Fn(x)p Fr(\))0 1547 y(is)f(equal)g(to)e(the)i (probabilit)o(y)g(that)e Fm(t)i Fr(equals)g(one,)f(and)g(this)h(latter) f(probabilit)o(y)h(can)g(b)q(e)g(expanded)g(via)f(Ba)o(y)o(es)0 1604 y(rule:)663 1673 y Fm(p)p Fr(\()p Fm(t)e Fr(=)h(1)p Fk(j)p Fn(x)p Fr(\))e(=)926 1642 y Fm(p)p Fr(\()p Fn(x)p Fk(j)p Fm(t)h Fr(=)h(1\))p Fm(p)p Fr(\()p Fm(t)f Fr(=)h(1\))p 926 1662 357 2 v 1061 1704 a Fm(p)p Fr(\()p Fn(x)p Fr(\))1892 1673 y(\(4\))0 1777 y(The)j(densit)o(y)g Fm(p)p Fr(\()p Fm(t)p Fk(j)p Fn(x)p Fr(\))f(in)h(this)g(equation)g(is)h(referred)e(to) g(as)h(the)f Fq(p)n(osterior)i(pr)n(ob)n(ability)j Fr(of)15 b(the)h(class)g(giv)o(en)g(the)0 1833 y(input,)21 b(and)f(the)f(densit) o(y)h Fm(p)p Fr(\()p Fn(x)p Fk(j)p Fm(t)p Fr(\))e(is)i(referred)g(to)f (as)g(the)g(the)g Fq(class-c)n(onditional)g(density)p Fr(.)32 b(Con)o(tin)o(uing)20 b(the)0 1890 y(deriv)m(ation,)c(w)o(e)f (expand)h(the)f(denominator)g(and)h(\(with)f(some)g(foresigh)o(t\))f (in)o(tro)q(duce)i(an)g(exp)q(onen)o(tial:)428 2008 y Fm(p)p Fr(\()p Fm(t)c Fr(=)h(1)p Fk(j)p Fn(x)p Fr(\))41 b(=)955 1978 y Fm(p)p Fr(\()p Fn(x)p Fk(j)p Fm(t)12 b Fr(=)h(1\))p Fm(p)p Fr(\()p Fm(t)f Fr(=)h(1\))p 749 1998 768 2 v 749 2039 a Fm(p)p Fr(\()p Fn(x)p Fk(j)p Fm(t)f Fr(=)h(1\))p Fm(p)p Fr(\()p Fm(t)f Fr(=)h(1\))d(+)g Fm(p)p Fr(\()p Fn(x)p Fk(j)p Fm(t)i Fr(=)h(0\))p Fm(p)p Fr(\()p Fm(t)f Fr(=)h(0\))668 2127 y(=)1107 2097 y(1)p 749 2117 738 2 v 749 2171 a(1)d(+)h(exp)905 2124 y Ff(n)932 2171 y Fk(\000)d Fr(ln)1021 2124 y Ff(h)1045 2148 y Fi(p)p Fj(\()p Fe(x)p Fl(j)p Fi(t)p Fj(=1\))p 1045 2160 137 2 v 1045 2187 a Fi(p)p Fj(\()p Fe(x)p Fl(j)p Fi(t)p Fj(=0\))1186 2124 y Ff(i)1216 2171 y Fk(\000)j Fr(ln)1307 2124 y Ff(h)1332 2148 y Fi(p)p Fj(\()p Fi(t)p Fj(=1\))p 1332 2160 104 2 v 1332 2187 a Fi(p)p Fj(\()p Fi(t)p Fj(=0\))1440 2124 y Ff(i)o(o)1892 2127 y Fr(\(5\))0 2271 y(W)l(e)k(see)h(that)e(the)i(p)q (osterior)f(probabilit)o(y)h(can)f(b)q(e)h(written)g(in)g(the)f(form)f (of)h(the)g Fq(lo)n(gistic)h(function)p Fr(:)844 2382 y Fm(y)f Fr(=)996 2351 y(1)p 934 2371 147 2 v 934 2413 a(1)10 b(+)g Fm(e)1033 2400 y Fl(\000)p Fi(z)1086 2382 y Fm(;)793 b Fr(\(6\))0 2494 y(where)12 b Fm(z)i Fr(is)e(a)g(function)h (of)e(the)h(lik)o(eliho)q(o)q(d)j(ratio)c Fm(p)p Fr(\()p Fn(x)p Fk(j)p Fm(t)h Fr(=)h(1\))p Fm(=p)p Fr(\()p Fn(x)p Fk(j)p Fm(t)e Fr(=)i(0\),)e(and)h(the)g(prior)g(ratio)f Fm(p)p Fr(\()p Fm(t)i Fr(=)g(1\))p Fm(=p)p Fr(\()p Fm(t)f Fr(=)0 2550 y(0\).)19 b(This)d(is)g(a)f(useful)h(represen)o(tation)f (of)g(the)g(p)q(osterior)g(probabilit)o(y)i(if)e Fm(z)j Fr(turns)d(out)g(to)f(b)q(e)i(simple.)964 2775 y(4)p eop %%Page: 3 27 3 26 bop 525 150 a 14208860 13498414 7828029 28878192 25589104 45915668 startTexFig 525 150 a %%BeginDocument: ps/mixture.ps save userdict /IslandDrawDict 300 dict dup begin put /ncpoint errordict /nocurrentpoint get def errordict begin /nocurrentpoint { dup /pathbbox load eq { pop 0 0 1 1 } { ncpoint } ifelse } bind def end /image_raster { %% sw sh dw dh xs ys translate scale /sh exch def /sw exch def /imagebuf sw 7 add 8 idiv string def sw sh 1 [sw 0 0 sh 0 0] { currentfile imagebuf readhexstring pop } image } bind def /m {moveto} bind def /l {lineto} bind def /c {curveto} bind def /n {newpath} bind def /cl {closepath} bind def /ar { %% sa ea sx sy rot tx ty matrix currentmatrix 8 1 roll translate rotate scale n 0 0 1 5 3 roll arc setmatrix } bind def /arn { %% sa ea sx sy rot tx ty matrix currentmatrix 8 1 roll translate rotate scale n 0 0 1 5 3 roll arcn setmatrix } bind def /el { %% sx sy rot tx ty matrix currentmatrix 6 1 roll translate rotate scale n 0 0 1 0 360 arc setmatrix cl } bind def /bp {setlinejoin setlinewidth setrgbcolor} bind def /bpbw {setlinejoin setlinewidth setgray} bind def /lw {setlinewidth} bind def /lj {setlinejoin} bind def /gr {setgray} bind def /BPSIDE 32 def %% pixels per pattern side /PATFREQ 3.0 def %% pattern pixels per mm /dp_mat [PATFREQ 0 0 PATFREQ 0 0] def /dp_pw BPSIDE def %% pattern pixel width /dp_ph BPSIDE def %% pattern pixel height /dp_w dp_pw PATFREQ div def %% pattern mm width /dp_h dp_ph PATFREQ div def %% pattern mm height /dp_bs 1 def %% pattern bits per pixel /savemat matrix def /topmat matrix def /patmat matrix def /patpath { /inv exch def topmat setmatrix pathbbox %% get lo - hi indecies /hy exch dp_h div floor cvi def /hx exch dp_w div floor cvi def /ly exch dp_h div floor cvi def /lx exch dp_w div floor cvi def lx 1 hx { dp_w mul ly 1 hy { dp_h mul exch dup 3 1 roll exch patmat currentmatrix pop translate dp_pw dp_ph inv dp_mat dp_proc imagemask patmat setmatrix } for pop } for } bind def /setpattern { /blue exch def /green exch def /red exch def /freq exch def /bwidth exch def /bpside exch def /bstring exch def /onbits 0 def /offbits 0 def freq 0 {/y exch def /x exch def /xindex x 1 add 2 div bpside mul cvi def /yindex y 1 add 2 div bpside mul cvi def bstring yindex bwidth mul xindex 8 idiv add get not 1 7 xindex 8 mod sub bitshift and 0 ne {/onbits onbits 1 add def 1} {/offbits offbits 1 add def 0} ifelse } setscreen {} settransfer systemdict /setcmykcolor known { /fact 1 onbits offbits onbits add div sub def 1 red sub fact mul 1 green sub fact mul 1 blue sub fact mul 0 setcmykcolor } { offbits offbits onbits add div setgray } ifelse } bind def /dmatrix matrix def /dpi 72 0 dmatrix defaultmatrix dtransform dup mul exch dup mul add sqrt def /B {gsave bp stroke grestore} bind def %% brush: gr lw lj /Bbw {gsave bpbw stroke grestore} bind def %% brush: gr lw lj /F {gsave setrgbcolor eofill grestore} bind def %% fill: gr /Fbw {gsave setgray eofill grestore} bind def %% fill: gr /PB {gsave setlinejoin setlinewidth setpattern stroke grestore} bind def /PF {gsave eoclip patpath grestore} bind def /BB { gsave setrgbcolor setlinejoin setlinewidth strokepath clip patpath grestore } bind def /BLACK { 0.0 } bind def /CP {closepath} bind def /FI {eofill} bind def /E {exch} bind def /FF {findfont} bind def /GR {grestore} bind def /GS {gsave} bind def /MF {makefont} bind def /NP {newpath} bind def /RO {rotate} bind def /ST {stroke} bind def /SC {scale} bind def /SF {setfont} bind def /SG {setgray} bind def /SLC {setlinecap} bind def /SLJ {setlinejoin} bind def /SLW {setlinewidth} bind def /TR {translate} bind def /WHITE { 1.0 } bind def /m {moveto} bind def /r {rmoveto} bind def /l {lineto} bind def /sp {x 0 rmoveto} bind def /rl {rlineto} bind def /s {show} bind def /box { NP m l l l CP } bind def /pageboundary { NP m l l l CP } bind def /BS { % black stroke GS SLJ SLW BLACK SG ST GR } bind def /WS { % white stroke GS SLJ SLW WHITE SG ST GR } bind def /reencode_small_dict 12 dict def /ReencodeSmall { reencode_small_dict begin /new_codes_and_names E def /new_font_name E def /base_font_name E def /base_font_dict base_font_name FF def /newfont base_font_dict maxlength dict def base_font_dict { E dup /FID ne { dup /Encoding eq { E dup length array copy newfont 3 1 roll put } { E newfont 3 1 roll put } ifelse } { pop pop } ifelse } forall newfont /FontName new_font_name put new_codes_and_names aload pop new_codes_and_names length 2 idiv { newfont /Encoding get 3 1 roll put } repeat new_font_name newfont definefont pop end %reencode_small_dict } def /extended_Zapf [ 8#223 /a89 8#224 /a90 8#225 /a93 8#226 /a94 8#227 /a91 8#230 /a92 8#231 /a205 8#232 /a85 8#233 /a206 8#234 /a86 8#235 /a87 8#236 /a88 8#237 /a95 8#240 /a96 ] def /extended_Standard [ 29 /thorn 30 /yacute 31 /divide 128 /Acircumflex 129 /Adieresis 130 /Agrave 131 /Aring 132 /Atilde 133 /Ccedilla 134 /Eacute 135 /Ecircumflex 136 /Edieresis 137 /Egrave 138 /Iacute 139 /Icircumflex 140 /Idieresis 141 /Igrave 142 /Ntilde 143 /Oacute 144 /Ocircumflex 145 /Odieresis 146 /Ograve 147 /Otilde 148 /Scaron 149 /Uacute 150 /Ucircumflex 151 /Udieresis 152 /Ugrave 153 /Ydieresis 154 /Zcaron 155 /aacute 156 /acircumflex 157 /adieresis 158 /agrave 159 /aring 160 /atilde 161 /exclamdown 162 /cent 163 /sterling 164 /fraction 165 /yen 166 /florin 167 /section 168 /currency 169 /quotesingle 170 /quotedblleft 171 /guillemotleft 172 /guilsinglleft 173 /guilsinglright 174 /fi 175 /fl 176 /plusminus 177 /endash 178 /dagger 179 /daggerdbl 180 /periodcentered 181 /twosuperior 182 /paragraph 183 /bullet 184 /quotesinglebase 185 /quotedblbase 186 /quotedblright 187 /guillemotright 188 /ellipsis 189 /perthousand 190 /threesuperior 191 /questiondown 192 /mu 193 /grave 194 /acute 195 /circumflex 196 /tilde 197 /macron 198 /breve 199 /dotaccent 200 /dieresis 201 /onesuperior 202 /ring 203 /cedilla 204 /onequarter 205 /hungarumlaut 206 /ogonek 207 /caron 208 /emdash 209 /ccedilla 210 /copyright 211 /eacute 212 /ecircumflex 213 /edieresis 214 /egrave 215 /iacute 216 /icircumflex 217 /idieresis 218 /igrave 219 /logicalnot 220 /minus 221 /ntilde 222 /oacute 223 /ocircumflex 224 /odieresis 225 /AE 226 /onehalf 227 /ordfeminine 228 /ograve 229 /otilde 230 /registered 231 /scaron 232 /Lslash 233 /Oslash 234 /OE 235 /ordmasculine 236 /trademark 237 /uacute 238 /ucircumflex 239 /udieresis 240 /ugrave 241 /ae 242 /ydieresis 243 /zcaron 244 /Aacute 245 /dotlessi 246 /threequarters 247 /Eth 248 /lslash 249 /oslash 250 /oe 251 /germandbls 252 /multiply 253 /Yacute 254 /Thorn 255 /eth ] def /extended_Symbol [ ] def /extend_font { % stack: fontname newfontname E dup (ZapfDingbats) eq { cvn E cvn extended_Zapf ReencodeSmall } { dup (Symbol) eq { cvn E cvn extended_Symbol ReencodeSmall } { cvn E cvn extended_Standard ReencodeSmall } ifelse } ifelse } bind def /getfont { /f E def f cvn where { begin f cvn load exec SF end } { f 0 f length 8 sub getinterval dup dup length 1 add string /localfont exch def localfont exch 0 exch putinterval localfont dup length 1 sub (X) putinterval localfont extend_font localfont FF /xsz f f length 4 sub 4 getinterval cvi def /ysz f f length 8 sub 4 getinterval cvi def [ xsz 0 0 ysz neg 0 0 ] MF dup f cvn E def SF } ifelse } bind def /ul { % space drop thickness GS currentpoint currentlinewidth currentpoint NP m 6 -3 roll SLW 0 E r 0 rl ST SLW m GR } bind def /ss { currentpoint pop E m } bind def /image_raster { % sw sh dw dh xs ys TR SC /sh E def /sw E def /imagebuf sw 7 add 8 idiv string def sw sh 1 [sw 0 0 sh 0 0] { currentfile imagebuf readhexstring pop } image } bind def /imagemask_raster { TR SC /sh E def /sw E def /imagebuf sw 7 add 8 idiv string def sw sh false [sw 0 0 sh 0 0] { currentfile imagebuf readhexstring pop } imagemask } bind def /image_color_raster { % sw sh sd dw dh xs ys systemdict /colorimage known not { /colorimage /colimg load def } if TR SC /sd E def /sh E def /sw E def /imagebuf sw 3 mul sd mul 7 add 8 idiv string def sw sh sd [sw 0 0 sh 0 0] { currentfile imagebuf readhexstring pop } false 3 colorimage } bind def /nx { /x E def } bind def 0. nx gsave 2.83465 -2.83465 scale 0 -279.4 translate topmat currentmatrix pop n 58.361 111.22 m 106.33 80.056 l 1 Fbw n 106.33 80.056 m 104 82.987 l 102.71 80.999 l cl 0 Fbw n 58.361 111.22 m 103.35 81.993 l gsave 0 0.352 0 Bbw grestore n 119.69 111.44 m 71.722 80.278 l 1 Fbw n 71.722 80.278 m 75.35 81.221 l 74.058 83.209 l cl 0 Fbw n 119.69 111.44 m 74.704 82.215 l gsave 0 0.352 0 Bbw grestore n 57.972 112.06 m 119.69 112.06 l 1 Fbw gsave [2 1 2 1] 0 setdash 0 0.352 0 Bbw grestore n 67.833 76.694 m 111.53 76.694 l 1 Fbw gsave [2 1 2 1] 0 setdash 0 0.352 0 Bbw grestore n 57.139 112.44 m 66.611 81.583 l 1 Fbw n 66.611 81.583 m 66.701 85.331 l 64.435 84.635 l cl 0 Fbw n 57.139 112.44 m 65.568 84.983 l gsave 0 0.352 0 Bbw grestore 3.0094 3.0094 0 57.75 111.83 el 1 Fbw gsave 0 0.352 0 Bbw grestore n 68.444 75.472 m 86.778 51.028 l 1 Fbw n 86.778 51.028 m 85.592 54.584 l 83.696 53.161 l cl 0 Fbw n 68.444 75.472 m 84.644 53.873 l gsave 0 0.352 0 Bbw grestore n 110.22 75.694 m 91.889 51.25 l 1 Fbw n 91.889 51.25 m 94.971 53.384 l 93.074 54.806 l cl 0 Fbw n 110.22 75.694 m 94.022 54.095 l gsave 0 0.352 0 Bbw grestore 4.7851 4.7851 0 67.833 76.847 el 1 Fbw gsave 0 0.352 0 Bbw grestore n 120.55 112.44 m 111.08 81.583 l 1 Fbw n 111.08 81.583 m 113.26 84.635 l 110.99 85.331 l cl 0 Fbw n 120.55 112.44 m 112.12 84.983 l gsave 0 0.352 0 Bbw grestore 3.0094 3.0094 0 119.94 111.83 el 1 Fbw gsave 0 0.352 0 Bbw grestore 4.7851 4.7851 0 109.86 76.847 el 1 Fbw gsave 0 0.352 0 Bbw grestore 4.7851 4.7851 0 89.139 46.819 el 1 Fbw gsave 0 0.352 0 Bbw grestore n savemat currentmatrix pop [1 0 0 1 58.3611 125.278] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (1) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1.11396 55.9167 124.102] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (x) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 121 124.444] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (n) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 118.167 123.444] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (x) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 41.8611 78.5278] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (P) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 47.9722 78.5278] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Bold03600360) getfont () s 0.00 SG (x) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 51.9444 79.4444] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Bold03600360) getfont () s 0.00 SG (|) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 45.5278 78.5833] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Roman03600360) getfont () s 0.00 SG (\() s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 57.75 78.6359] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Roman03600360) getfont () s 0.00 SG (\)) s savemat setmatrix n savemat currentmatrix pop [1.08202 0 0 1 53.8814 78.8333] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (1) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 130.167 78.6111] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (M) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 117.556 78.5278] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (P) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 123.667 78.5278] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Bold03600360) getfont () s 0.00 SG (x) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 127.639 79.4444] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Bold03600360) getfont () s 0.00 SG (|) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 121.222 78.5833] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Roman03600360) getfont () s 0.00 SG (\() s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 136.111 78.4444] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Roman03600360) getfont () s 0.00 SG (\)) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 104.417 62.0278] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (M) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 100.75 60.1944] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Symbol03600360) getfont () s 0.00 SG (p) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 70.8889 60.1944] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Symbol03600360) getfont () s 0.00 SG (p) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 74.25 62.0278] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (1) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 83.0278 38.1111] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic03600360) getfont () s 0.00 SG (P) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 90.6667 38.1111] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Bold03600360) getfont () s 0.00 SG (x) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 94.6389 38.1666] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Roman03600360) getfont () s 0.00 SG (\)) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 87.3889 38.1666] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Roman03600360) getfont () s 0.00 SG (\() s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 71.5 92.8056] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Symbol03600360) getfont () s 0.00 SG (m) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 75.0833 94.5556] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (1) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 100.75 92.8056] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Symbol03600360) getfont () s 0.00 SG (m) s savemat setmatrix n savemat currentmatrix pop [1 0 0 1 104.639 94.3333] concat 25.4 1440 div 1.000000 mul dup scale 0 0 m 0 0 m 0 ss (Times-Italic02800280) getfont () s 0.00 SG (M) s savemat setmatrix n 101.44 83.417 m 105.11 90.444 112.44 87.083 112.44 87.083 c gsave 0 0.176 0 Bbw grestore n 76.306 83.417 m 72.639 90.444 65.306 87.083 65.306 87.083 c gsave 0 0.176 0 Bbw grestore userdict /#copies 1 put showpage grestore end restore %%EndDocument endTexFig 0 1103 a Fr(Figure)19 b(1:)27 b(A)19 b(net)o(w)o(ork)e(represen)o (tation)i(of)f(a)h(Gaussian)g(mixture)g(distribution.)32 b(The)19 b(input)h(pattern)e Fn(x)g Fr(is)0 1159 y(represen)o(ted)f(b)o (y)e(n)o(umerical)j(v)m(alues)f(asso)q(ciated)f(with)h(the)f(input)h (no)q(des)f(in)h(the)f(lo)o(w)o(er)g(lev)o(el.)24 b(Eac)o(h)16 b(link)h(has)0 1216 y(a)d(w)o(eigh)o(t)g Fm(\026)207 1223 y Fi(ij)238 1216 y Fr(,)g(whic)o(h)h(is)g(the)f Fm(j)538 1199 y Fi(th)587 1216 y Fr(comp)q(onen)o(t)h(of)e(the)i(mean)f (v)o(ector)g(for)g(the)g Fm(i)1365 1199 y Fi(th)1414 1216 y Fr(Gaussian.)20 b(The)14 b Fm(i)1735 1199 y Fi(th)1784 1216 y Fr(in)o(terme-)0 1272 y(diate)i(no)q(de)h(con)o(tains)f(the)g (co)o(v)m(ariance)h(matrix)e Fg(\006)895 1279 y Fi(i)925 1272 y Fr(and)h(calculates)h(the)f(Gaussian)g(conditional)i(probabilit) o(y)0 1329 y Fm(p)p Fr(\()p Fn(x)p Fk(j)p Fm(i;)8 b Fg(\026)149 1340 y Fi(i)163 1329 y Fm(;)g Fg(\006)226 1336 y Fi(i)240 1329 y Fr(\).)20 b(These)15 b(probabilities)j(are)d(w)o(eigh)o(ted)g(b) o(y)g(the)h(mixing)g(prop)q(ortions)f Fm(\031)1510 1336 y Fi(i)1539 1329 y Fr(and)h(the)f(output)g(no)q(de)0 1385 y(calculates)h(the)g(w)o(eigh)o(ted)f(sum)g Fm(p)p Fr(\()p Fn(x)p Fr(\))d(=)718 1353 y Ff(P)761 1397 y Fi(i)783 1385 y Fm(\031)809 1392 y Fi(i)823 1385 y Fm(p)p Fr(\()p Fn(x)p Fk(j)p Fm(i;)c Fg(\026)972 1396 y Fi(i)986 1385 y Fm(;)g Fg(\006)1049 1392 y Fi(i)1063 1385 y Fr(\).)0 1521 y(in)19 b(turn)f(pro)o(vides)h(us)f(with)h(the)f(necessary)h (information)f(to)g(estimate)g(the)g(\\in)o(v)o(erse")g(conditional)i (densit)o(y)0 1578 y Fm(p)p Fr(\()p Fn(x)p Fk(j)p Fn(t)p Fr(\).)f(The)c(calculation)i(of)e(suc)o(h)g(in)o(v)o(erses)h(is)f(imp)q (ortan)o(t)g(for)g(applications)i(in)f(con)o(trol)e(and)i (optimization.)91 1634 y(A)c(general)h(and)g(\015exible)i(approac)o(h)d (to)g(densit)o(y)h(estimation)g(is)g(to)e(treat)h(the)h(densit)o(y)g (as)f(b)q(eing)i(comp)q(osed)0 1691 y(of)j(a)h(set)f(of)h Fm(M)k Fr(simpler)d(densities.)30 b(This)18 b(approac)o(h)f(in)o(v)o (olv)o(es)i(mo)q(deling)g(the)f(observ)o(ed)f(data)h(as)f(a)g(sample)0 1747 y(from)d(a)h Fq(mixtur)n(e)i(density)p Fr(:)719 1840 y Fm(p)p Fr(\()p Fn(x)p Fk(j)p Fn(w)q Fr(\))11 b(=)928 1787 y Fi(M)917 1799 y Ff(X)918 1890 y Fi(i)p Fj(=1)984 1840 y Fm(\031)1010 1847 y Fi(i)1024 1840 y Fm(p)p Fr(\()p Fn(x)p Fk(j)p Fm(i;)d Fn(w)1181 1847 y Fi(i)1193 1840 y Fr(\))p Fm(;)668 b Fr(\(2\))0 1960 y(where)21 b(the)f Fm(\031)246 1967 y Fi(i)281 1960 y Fr(are)g(constan)o(ts)g(kno)o(wn)g (as)g Fq(mixing)h(pr)n(op)n(ortions)p Fr(,)h(and)f(the)f Fm(p)p Fr(\()p Fn(x)p Fk(j)p Fm(i;)8 b Fn(w)1525 1967 y Fi(i)1537 1960 y Fr(\))20 b(are)g(the)h Fq(c)n(omp)n(onent)0 2016 y(densities)p Fr(,)14 b(generally)k(tak)o(en)d(to)g(b)q(e)i(from)e (a)h(simple)h(parametric)f(family)l(.)23 b(A)16 b(common)f(c)o(hoice)i (of)f(comp)q(onen)o(t)0 2073 y(densit)o(y)d(is)g(the)g(m)o(ultiv)m (ariate)g(Gaussian,)g(in)g(whic)o(h)h(case)e(the)h(parameters)e Fn(w)1348 2080 y Fi(i)1374 2073 y Fr(are)i(the)f(means)h(and)f(co)o(v)m (ariance)0 2129 y(matrices)19 b(of)f(eac)o(h)i(of)e(the)h(comp)q(onen)o (ts.)31 b(By)19 b(v)m(arying)h(the)f(means)g(and)g(co)o(v)m(ariances)h (to)e(place)i(and)f(orien)o(t)0 2186 y(the)13 b(Gaussians)g (appropriately)l(,)h(a)e(wide)i(v)m(ariet)o(y)f(of)g(high-dimensional,) j(m)o(ulti-mo)q(dal)e(data)e(can)h(b)q(e)h(mo)q(deled.)0 2242 y(This)i(approac)o(h)f(to)f(densit)o(y)i(estimation)f(is)h(essen)o (tially)h(a)e(probabilistic)i(form)e(of)f(clustering.)91 2299 y(Gaussian)19 b(mixtures)h(ha)o(v)o(e)f(a)g(represen)o(tation)h (as)f(a)g(net)o(w)o(ork)f(diagram)h(as)g(sho)o(wn)h(in)g(Figure)f(1.)33 b(The)0 2355 y(utilit)o(y)20 b(of)e(suc)o(h)h(net)o(w)o(ork)f(represen) o(tations)g(will)j(b)q(ecome)e(clearer)g(as)g(w)o(e)f(pro)q(ceed;)j (for)d(no)o(w,)h(it)g(su\016ces)g(to)0 2412 y(note)c(that)g(not)g(only) h(mixture)g(mo)q(dels,)g(but)g(also)f(a)g(wide)i(v)m(ariet)o(y)e(of)g (other)g(classical)i(statistical)f(mo)q(dels)h(for)0 2468 y(densit)o(y)12 b(estimation)g(are)f(represen)o(table)i(as)e (simple)i(net)o(w)o(orks)e(with)g(one)h(or)f(more)g(la)o(y)o(ers)h(of)f (adaptiv)o(e)h(w)o(eigh)o(ts.)0 2524 y(These)j(metho)q(ds)f(include)j Fq(princip)n(al)e(c)n(omp)n(onent)f(analysis)p Fr(,)f Fq(c)n(anonic)n(al)h(c)n(orr)n(elation)h(analysis)p Fr(,)e Fq(kernel)i(density)964 2775 y Fr(3)p eop %%Page: 2 28 2 27 bop 0 192 a Fr(al.,)18 b(1991].)27 b(The)19 b(net)o(w)o(ork)e (input)i Fn(x)f Fr(consists)g(of)f(one)i(in-fo)q(cus)g(and)f(one)h (de-fo)q(cused)g(image)f(of)g(a)g(star)f(and)0 248 y(the)f(output)f Fn(t)h Fr(consists)g(of)g(a)f(set)h(of)f(co)q(e\016cien)o(ts)i(that)e (describ)q(e)i(the)f(phase)g(distortion)g(due)h(to)e(atmospheric)0 304 y(turbulence.)20 b(These)12 b(output)g(v)m(alues)g(are)g(then)g (used)g(to)f(mak)o(e)g(real-time)h(adjustmen)o(ts)f(of)g(the)h(m)o (ultiple)h(mirror)0 361 y(segmen)o(ts)i(to)f(cancel)j(the)e (atmospheric)g(distortion.)91 417 y(Classi\014cation)e(and)f (regression)g(problems)g(can)g(also)g(b)q(e)g(view)o(ed)h(as)e(sp)q (ecial)j(cases)d(of)h(densit)o(y)g(estimation.)0 474 y(The)21 b(most)e(general)i(and)f(complete)h(description)h(of)e(the)g (data)g(is)g(giv)o(en)h(b)o(y)f(the)h(probabilit)o(y)g(distribution)0 530 y(function)15 b Fm(p)p Fr(\()p Fn(x)p Fm(;)8 b Fn(t)p Fr(\))k(in)j(the)f(join)o(t)g(input-target)g(space.)20 b(Ho)o(w)o(ev)o(er,)12 b(the)i(usual)h(goal)f(is)g(to)g(b)q(e)g(able)h (to)f(mak)o(e)f(go)q(o)q(d)0 587 y(predictions)j(for)e(the)g(target)g (v)m(ariables)h(when)g(presen)o(ted)g(with)g(new)g(v)m(alues)g(of)f (the)h(inputs.)20 b(In)15 b(this)g(case)g(it)f(is)0 643 y(con)o(v)o(enien)o(t)i(to)e(decomp)q(ose)i(the)f(join)o(t)g (distribution)i(in)f(the)g(form:)779 745 y Fm(p)p Fr(\()p Fn(x)p Fm(;)8 b Fn(t)p Fr(\))j(=)i Fm(p)p Fr(\()p Fn(t)p Fk(j)p Fn(x)p Fr(\))p Fm(p)p Fr(\()p Fn(x)p Fr(\))719 b(\(1\))0 847 y(and)16 b(to)e(consider)j(only)f(the)f(conditional)i (distribution)h Fm(p)p Fr(\()p Fn(t)p Fk(j)p Fn(x)p Fr(\),)13 b(in)k(other)e(w)o(ords)f(the)i(distribution)h(of)e Fn(t)h Fq(given)0 904 y Fr(the)g(v)m(alue)g(of)f Fn(x)p Fr(.)21 b(Th)o(us)15 b(classi\014cation)i(and)f(regression)f(in)o(v)o(olv)o(e)h (the)g(estimation)g(of)f Fq(c)n(onditional)f Fr(densities,)j(a)0 960 y(problem)f(whic)o(h)g(has)f(its)g(o)o(wn)g(idiosyncracies.)91 1017 y(The)k(organization)g(of)f(the)h(c)o(hapter)f(is)h(as)g(follo)o (ws.)30 b(In)19 b(Section)h(2)e(w)o(e)h(presen)o(t)f(examples)i(of)e (net)o(w)o(ork)0 1073 y(represen)o(tations)e(of)f(unconditional)j(and)e (conditional)h(densities.)23 b(In)16 b(Section)h(3)e(w)o(e)g(discuss)i (the)f(problem)g(of)0 1130 y(adjusting)j(the)h(parameters)e(of)h(these) g(net)o(w)o(orks)f(to)g(\014t)h(them)g(to)g(data.)31 b(This)19 b(problem)h(has)f(a)g(n)o(um)o(b)q(er)g(of)0 1186 y(practical)f(asp)q(ects,)f(including)j(the)d(c)o(hoice)h(of)f (optimization)h(pro)q(cedure)g(and)f(the)g(metho)q(d)g(used)h(to)e(con) o(trol)0 1243 y(net)o(w)o(ork)j(complexit)o(y)l(.)35 b(W)l(e)21 b(then)f(discuss)h(a)f(broader)f(p)q(ersp)q(ectiv)o(e)j(on)e (probabilistic)i(net)o(w)o(ork)d(mo)q(dels)i(in)0 1299 y(Section)16 b(4.)k(The)15 b(\014nal)h(section)g(presen)o(ts)f(further) g(information)g(and)h(p)q(oin)o(ters)f(to)g(the)g(literature.)0 1442 y Fp(2)69 b(Represen)n(tation)0 1544 y Fr(In)20 b(this)h(section)f(w)o(e)g(describ)q(e)h(a)f(selection)h(of)e(neural)i (net)o(w)o(ork)e(arc)o(hitectures)h(that)f(ha)o(v)o(e)h(b)q(een)h(prop) q(osed)0 1600 y(as)c(represen)o(tations)g(for)g(unconditional)j(and)d (conditional)i(densities.)28 b(After)17 b(a)g(brief)h(discussion)h(of)e (densit)o(y)0 1657 y(estimation,)i(w)o(e)f(discuss)i(classi\014cation)f (and)g(regression,)g(b)q(eginning)h(with)f(simple)h(mo)q(dels)f(that)f (illustrate)0 1713 y(the)j(fundamen)o(tal)h(ideas)g(and)f(then)h (progressing)f(to)f(more)h(complex)h(arc)o(hitectures.)38 b(W)l(e)21 b(fo)q(cus)g(here)h(on)0 1770 y(represen)o(tational)16 b(issues,)f(p)q(ostp)q(oning)h(the)g(problem)g(of)e(learning)j(from)d (data)h(un)o(til)h(the)f(follo)o(wing)h(section.)0 1891 y Fh(2.1)56 b(Densit)n(y)17 b(estimation)0 1977 y Fr(W)l(e)d(b)q(egin)i (with)f(a)f(brief)h(discussion)h(of)d(densit)o(y)i(estimation,)g (utilizing)i(the)d(Gaussian)g(mixture)h(mo)q(del)g(as)f(an)0 2034 y(illustrativ)o(e)j(mo)q(del.)j(W)l(e)c(return)f(to)f(more)h (complex)h(densit)o(y)g(estimation)f(tec)o(hniques)i(later)e(in)h(the)f (c)o(hapter.)91 2090 y(Although)k(densit)o(y)g(estimation)g(can)g(b)q (e)g(the)g(main)g(goal)f(of)g(a)h(learning)g(system,)g(as)f(in)i(the)e (diagnosis)0 2146 y(example)d(men)o(tioned)g(in)g(the)f(in)o(tro)q (duction,)h(densit)o(y)g(estimation)g(mo)q(dels)g(arise)f(more)g(often) g(as)g(comp)q(onen)o(ts)0 2203 y(of)19 b(the)h(solution)g(to)e(a)i (more)f(general)g(classi\014cation)i(or)e(regression)h(problem.)33 b(T)l(o)19 b(return)h(to)e(Eq.)h(1,)h(note)0 2259 y(that)c(the)h(join)o (t)f(densit)o(y)i(is)f(comp)q(osed)g(of)f Fm(p)p Fr(\()p Fn(t)p Fk(j)p Fn(x)p Fr(\),)g(to)g(b)q(e)h(handled)i(b)o(y)d (classi\014cation)i(or)f(regression)f(mo)q(dels,)0 2316 y(and)21 b Fm(p)p Fr(\()p Fn(x)p Fr(\),)f(the)h(\(unconditional\))h (input)g(densit)o(y)l(.)37 b(There)21 b(are)f(sev)o(eral)h(reasons)f (for)g(w)o(an)o(ting)g(to)g(form)g(an)0 2372 y(explicit)e(mo)q(del)e (of)f(the)h(input)g(densit)o(y)l(.)22 b(First,)15 b(real-life)i(data)e (sets)g(often)g(ha)o(v)o(e)h(missing)g(comp)q(onen)o(ts)g(in)g(the)0 2429 y(input)g(v)o(ector.)21 b(Ha)o(ving)15 b(a)g(mo)q(del)i(of)e(the)g (densit)o(y)h(allo)o(ws)g(the)g(missing)g(comp)q(onen)o(ts)g(to)e(b)q (e)j(\\\014lled)g(in")f(in)g(an)0 2485 y(in)o(telligen)o(t)h(w)o(a)o(y) l(.)j(This)c(can)g(b)q(e)g(useful)h(b)q(oth)e(for)g(training)h(and)g (for)e(prediction)k([cf.)c(Bishop,)i(1995].)j(Second,)0 2542 y(as)13 b(w)o(e)f(see)i(in)g(Eq.)e(1,)h(a)g(mo)q(del)h(of)e Fm(p)p Fr(\()p Fn(x)p Fr(\))g(mak)o(es)h(p)q(ossible)i(an)e(estimate)g (of)f(the)h(join)o(t)g(probabilit)o(y)i Fm(p)p Fr(\()p Fn(x)p Fm(;)8 b Fn(t)p Fr(\).)17 b(Th)o(us)964 2775 y(2)p eop %%Page: 1 29 1 28 bop 0 192 a Fp(1)69 b(In)n(tro)r(duction)0 293 y Fr(Within)15 b(the)f(broad)g(scop)q(e)g(of)g(the)g(study)g(of)g (arti\014cial)h(in)o(telligence,)h(researc)o(h)e(in)h(neural)g(net)o(w) o(orks)e(is)h(c)o(harac-)0 349 y(terized)i(b)o(y)f(a)g(particular)h(fo) q(cus)f(on)g(pattern)g(recognition)h(and)f(pattern)g(generation.)20 b(Man)o(y)14 b(neural)i(net)o(w)o(ork)0 406 y(metho)q(ds)g(can)g(b)q(e) g(view)o(ed)g(as)g(generalizations)h(of)e(classical)i(pattern-orien)o (ted)f(tec)o(hniques)h(in)f(statistics)g(and)0 462 y(the)c(engineering) j(areas)c(of)h(signal)h(pro)q(cessing,)h(system)e(iden)o(ti\014cation)i (and)e(con)o(trol)g(theory)l(.)19 b(As)12 b(in)i(these)e(par-)0 519 y(en)o(t)17 b(discipline)q(s,)j(the)d(notion)g(of)g(\\pattern")f (in)i(neural)g(net)o(w)o(ork)e(researc)o(h)h(is)h(essen)o(tially)g (probabilistic)i(and)0 575 y(n)o(umerical.)26 b(Neural)17 b(net)o(w)o(ork)f(metho)q(ds)h(ha)o(v)o(e)f(had)h(their)h(greatest)d (impact)i(in)h(problems)g(where)f(statistical)0 632 y(issues)f (dominate)g(and)f(where)g(data)g(are)g(easily)h(obtained.)91 688 y(A)c(neural)h(net)o(w)o(ork)e(is)i(\014rst)f(and)g(foremost)f(a)h (graph,)g(with)g(patterns)g(represen)o(ted)g(in)h(terms)f(of)g(n)o (umerical)0 745 y(v)m(alues)23 b(attac)o(hed)d(to)h(the)h(no)q(des)g (of)f(the)g(graph,)h(and)g(transformations)e(b)q(et)o(w)o(een)i (patterns)e(ac)o(hiev)o(ed)j(via)0 801 y(simple)13 b(message-passing)e (algorithms.)18 b(Man)o(y)11 b(neural)g(net)o(w)o(ork)f(arc)o (hitectures,)i(ho)o(w)o(ev)o(er,)f(are)g(also)g(statistical)0 858 y(pro)q(cessors,)g(c)o(haracterized)g(b)o(y)g(making)g(particular)g (probabilistic)i(assumptions)e(ab)q(out)f(data.)18 b(As)10 b(w)o(e)h(will)h(see,)0 914 y(this)17 b(conjunction)h(of)e(graphical)i (algorithms)f(and)g(probabilit)o(y)h(theory)e(is)h(not)g(unique)h(to)e (neural)h(net)o(w)o(orks,)0 970 y(but)e(c)o(haracterizes)f(a)g(wider)h (family)h(of)e(probabilistic)i(systems)e(in)h(the)g(form)f(of)g(c)o (hains,)h(trees,)f(and)g(net)o(w)o(orks)0 1027 y(that)g(are)h(curren)o (tly)h(b)q(eing)h(studied)f(throughout)e(AI)i([Spiegelhalter,)h(et)e (al.,)f(1993].)91 1083 y(Neural)g(net)o(w)o(orks)f(ha)o(v)o(e)g(found)h (a)g(wide)g(range)g(of)f(applications,)i(the)f(ma)s(jorit)o(y)e(of)i (whic)o(h)g(are)g(asso)q(ciated)0 1140 y(with)21 b(problems)h(in)f (pattern)g(recognition)g(and)g(con)o(trol)g(theory)l(.)36 b(In)22 b(this)f(con)o(text,)g(neural)h(net)o(w)o(orks)e(can)0 1196 y(b)q(est)c(b)q(e)h(view)o(ed)f(as)g(a)f(class)i(of)e(algorithms)h (for)f(statistical)h(mo)q(deling)i(and)e(prediction.)23 b(Based)17 b(on)e(a)h(source)0 1253 y(of)g Fq(tr)n(aining)h(data)p Fr(,)g(the)g(aim)f(is)h(to)f(pro)q(duce)i(a)e(statistical)h(mo)q(del)g (of)f(the)h(pro)q(cess)g(from)e(whic)o(h)j(the)e(data)g(are)0 1309 y(generated,)c(so)e(as)h(to)g(allo)o(w)g(the)h(b)q(est)f (predictions)i(to)e(b)q(e)g(made)h(for)e(new)i(data.)18 b(W)l(e)11 b(shall)h(\014nd)g(it)g(con)o(v)o(enien)o(t)g(to)0 1366 y(distinguish)h(three)e(broad)g(t)o(yp)q(es)g(of)g(statistical)h (mo)q(deling)g(problem,)g(whic)o(h)g(w)o(e)f(shall)h(call)h Fq(density)f(estimation)p Fr(,)0 1422 y Fq(classi\014c)n(ation)h Fr(and)j Fq(r)n(e)n(gr)n(ession)p Fr(.)91 1479 y(F)l(or)d(densit)o(y)i (estimation)g(problems)f(\(also)g(referred)g(to)g(as)g Fq(unsup)n(ervise)n(d)g(le)n(arning)j Fr(problems\),)d(the)g(goal)0 1535 y(is)g(to)e(mo)q(del)j(the)e(unconditional)i(distribution)g(of)e (data)g(describ)q(ed)i(b)o(y)e(some)g(v)o(ector)g Fn(x)p Fr(.)19 b(A)13 b(practical)h(example)0 1591 y(of)f(the)g(application)h (of)f(densit)o(y)g(estimation)h(in)o(v)o(olv)o(es)f(the)g(in)o (terpretation)g(of)g(X-ra)o(y)f(images)h(\(mammograms\))0 1648 y(used)e(for)g(breast)f(cancer)h(screening)h([T)l(arassenk)o(o,)e (1995].)17 b(In)12 b(this)f(case)g(the)g(training)g(v)o(ectors)f Fn(x)h Fr(form)f(a)g(sample)0 1704 y(tak)o(en)k(from)h(normal)f (\(non-cancerous\))h(images,)g(and)g(a)g(net)o(w)o(ork)e(mo)q(del)j(is) g(used)f(to)f(build)j(a)e(represen)o(tation)0 1761 y(of)g(the)g(densit) o(y)h Fm(p)p Fr(\()p Fn(x)p Fr(\).)i(When)e(a)f(new)g(input)h(v)o (ector)e Fn(x)954 1744 y Fl(0)981 1761 y Fr(is)h(presen)o(ted)h(to)e (the)i(system,)e(a)h(high)h(v)m(alue)g(for)f Fm(p)p Fr(\()p Fn(x)1922 1744 y Fl(0)1932 1761 y Fr(\))0 1817 y(indicates)g(a)f (normal)f(image)h(while)h(a)f(lo)o(w)f(v)m(alue)i(indicates)h(a)d(no)o (v)o(el)h(input)h(whic)o(h)f(migh)o(t)g(b)q(e)g(c)o(haracteristic)g(of) 0 1874 y(an)g(abnormalit)o(y)l(.)20 b(This)15 b(is)g(used)h(to)e(lab)q (el)i(regions)e(of)h(images)f(whic)o(h)i(are)e(un)o(usual,)h(for)f (further)g(examination)0 1930 y(b)o(y)h(an)g(exp)q(erienced)j (clinician.)91 1987 y(F)l(or)d(classi\014cation)i(and)e(regression)h (problems)g(\(often)f(referred)g(to)g(as)g Fq(sup)n(ervise)n(d)h(le)n (arning)i Fr(problems\),)0 2043 y(w)o(e)e(need)i(to)e(distinguish)i(b)q (et)o(w)o(een)f Fq(input)g Fr(v)m(ariables,)h(whic)o(h)f(w)o(e)f(again) h(denote)g(b)o(y)f Fn(x)p Fr(,)g(and)h Fq(tar)n(get)k Fr(v)m(ariables)0 2100 y(whic)o(h)h(w)o(e)f(denote)g(b)o(y)g(the)h(v)o (ector)e Fn(t)p Fr(.)38 b(Classi\014cation)22 b(problems)g(require)g (that)f(eac)o(h)g(input)h(v)o(ector)f Fn(x)f Fr(b)q(e)0 2156 y(assigned)14 b(to)f(one)h(of)g Fm(C)i Fr(classes)e Fk(C)581 2163 y Fj(1)601 2156 y Fm(;)8 b(:)g(:)g(:)d(;)j Fk(C)727 2163 y Fi(C)755 2156 y Fr(,)14 b(in)h(whic)o(h)f(case)g(the)g (target)e(v)m(ariables)j(represen)o(t)f(class)g(lab)q(els.)21 b(As)0 2212 y(an)c(example,)g(consider)h(the)f(problem)g(of)f (recognizing)i(handwritten)f(digits)h([LeCun,)f(et)g(al.,)f(1989].)23 b(In)18 b(this)0 2269 y(case)h(the)g(input)g(v)o(ector)g(w)o(ould)g(b)q (e)g(some)g(\(pre-pro)q(cessed\))g(image)g(of)f(the)h(digit,)h(and)f (the)g(net)o(w)o(ork)f(w)o(ould)0 2325 y(ha)o(v)o(e)d(ten)h(outputs,)f (one)g(for)g(eac)o(h)h(digit,)g(whic)o(h)g(can)g(b)q(e)g(used)g(to)f (assign)g(input)i(v)o(ectors)e(to)f(the)i(appropriate)0 2382 y(class)g(\(as)e(discussed)j(in)f(Section)g(2\).)91 2438 y(Regression)d(problems)f(in)o(v)o(olv)o(e)h(estimating)f(the)g(v) m(alues)i(of)d(con)o(tin)o(uous)i(v)m(ariables.)20 b(F)l(or)11 b(example,)i(neural)0 2495 y(net)o(w)o(orks)j(ha)o(v)o(e)h(b)q(een)h (used)g(as)f(part)f(of)h(the)h(con)o(trol)e(system)h(for)g(adaptiv)o(e) g(optics)h(telescop)q(es)g([Sandler,)g(et)964 2775 y(1)p eop %%Page: 0 30 0 29 bop 0 192 a Fr(In)16 b(press:)k(A.)15 b(T)l(uc)o(k)o(er,)f (\(Ed.\))20 b Fq(CR)o(C)15 b(Handb)n(o)n(ok)h(of)g(Computer)h(Scienc)n (e)p Fr(,)c(CR)o(C)i(Press,)g(Bo)q(ca)g(Raton,)g(FL.)719 397 y Fp(Neural)23 b(Net)n(w)n(orks)787 505 y Fo(Mic)o(hael)15 b(I.)g(Jordan)568 563 y(Massac)o(h)o(usetts)h(Institute)g(of)g(T)l(ec)o (hnology)731 671 y(Christopher)g(M.)g(Bishop)795 728 y(Aston)g(Univ)o(ersit)o(y)801 834 y Fq(January)h(18,)g(1996)p eop %%Trailer end userdict /end-hook known{end-hook}if %%EOF </j)990>