Transcript
Probabilistic Question Answering on the Web Dragomir Radev, Weiguo Fan, Hong Qi, Harris Wu, Amardeep Grewal The University of Michigan Ann Arbor, MI, 48105 USA
ABSTRACT
! "
# $ % & # '(()* # +, )-. / &
0$)
Categories and Subject Descriptors 12 3 4" 5 6 7+ 3 4" 0 - 6 7+/ 3
4" 5 8
General Terms 9 7 - (
Keywords
1.
INTRODUCTION TO WEB-BASED Q&A
9 : ;90
- < = >? <
Copyright is held by the author/owner(s). , May 7–11, 2002, Honolulu, Hawaii, USA. ACM 1-58113-449-5/02/0005.
9 >? <' &) &) * 97 '@= :*? #
# A 9 ' A9* $ ( ( ) # '(()* $ B 5
0$) ' < ?*
A9 () 9 # C 5 3D4 $ 3+4 9 # C : 5 $ ! :
< ? < ?
!
A9 ' * 1 ; # < 9 >? # ' *
! "# $" $ % # &&& '% ( ) * + , # "# - "" & #& & &
"EE #
"EE
"EE "/2,,E
"EE
$ # $ ! :
#
9 9 ; D 9 # A9 (() '( ( ) # * # (()
# #
' * ' * #
0 9 9 0 F +,,D
2.
RELATED WORK
0 9) 3G4 ! A9 1 5$ $ 9 # 9 5B)9H 324 # 5B)9H 9 A9 ( " )-. AI9 3J4 6
#
2.1 Related work from TREC
)-. 3J4 0 )-.
+K
3L M4 #
#
# #
3/4 AI9 $) ( 9 ! #
#
# ( #
! 9 3+4 ! ;
! ! 9
K
! ! & # .# 3N4
# O $7; #
9
#
#
9 % % # 3D,4 ;
(
P !
; )-. AI9 3J4
2.2 Related work on the Web
)-. % A 9 1 ! )-. Q % AI9 8 & 3DD4 # % 9 3D+4
!
9 3DG4
K
! ! 9 #
$
9 #
3D24 9 DJ ! -5
% ! 9 AI9
3D4 0 )-. ;
$ #
! #
! " 0 3D+ M4
O $7; ' *
& % 3D4 9
5 E $ 5 D,, # '& -O 7 0 ( . +,,+* # $
! # 0 # 9 % #
3. A PROBABILISTIC APPROACH TO PHRASE RERANKING 3.1 The general architecture of our system & "
8
3D24 $ :
8
" O
8
# ' *
' * 8
8
' * # 8 #
3.2 Evaluation metric 9 # ' * # ' * "
R
D
'D*
'
* ; D, " S S R M+J )7) ! $ # ' *
)7) )-. % )-. #
!
B )7) )-. # ! 9 )-. P % ' * # ' P !*
< ? 3J4 $ )-. / N D, $0 3J4
D,,, E B
+,, )-./
5 %
! ! )-.
3.3 Manual vs. automatic evaluation consistency
; ? . DM 2 O " ) 3DJ4 D+,, )-./ )-.N )-.D,
!
; < $ 5 ? # ! <
.1B= 3DL4 & ( 9 2D
#
: ) ) ) 4 ) ) 4 5) # +5 # +52 +53 +5. +52 +56 +50 +5+" +" 1+.
0 ) ! A )-.N )-./ )-.D,
$ GM ; < - 1"2? < ? < ? O < ?
) ; )
< - 1 J , $; ' $ $=- * 97 ' $=- *? 0 ) # <1 - ( - ?
) ) G
$ ! % & ! ' #
$0/ $0/ ! $0/ ! $0/ > $0/ ! >
;;<= ?>'= ;>@ =
;<= '>=
" #
./01+ .2 / 3$/ +-4/0 3/5+5$51+ 106+57$51+ 3/ 05.$51+ 440/85$51+ *+1)+10 0$/ 2/+6$9 1+/ 0/1+ 3-0$51+ .-0.1/ +15+2 1$9/0 <7 ? # < C >?6 # < >? <( ? < ?
! O A <& ?
4.1 Machine learning approach ) O $ DG N
4.2 Heuristic algorithm
! O A O 1 < ? 2/2 )-.N 9 2 < ? < ? < ? ; # J K E $ K : (&0 3DM4 E
7 ' * <
B 0 >?
( ' ' '
A >;B A@>B A<>B #A B & A;''B AB 0/1+A B +15+2A?!B .2 /A;?B 3/5+5$51+A;@B ./01+A !B 106A @B +-4/0A B +-4/0A''B 2/+6$9A@B 0$/A;B 1+/A;B 3-0$51+A'B 0/1+A B 3/ 1$9/0A B
)
! )-.
. + 3 A B 1
( ' * < C >?
7 < ?
; < E7 E E EQKT . E( E0 >E? ! < ? < ? 6 O <&)9 $T9 $&? . 7 < ?
( # ; O # < U >? 6 <7-;$$ $&? 6
&
! ! ; < :
>? < ? <
? < ? 7 % # < ! )
#>? < : >?
# < ! C ?
:
F - 1 0 < ? O - 1
L ! # )-.N 6 )-./ )-.N 6
* ! % & ' #
$0/ $0/ ?!= $0/ ! ?<= $0/ ! > <@=
$0/ ! $0/ > C= != @= !;= CC= ?@=
) +,V G,V )-./N )-.D, 9 L /V D/V $
O ; # < ?
< ( ( DNND? O < 0 DN/+?
5. DOCUMENT RETRIEVAL W P
9
' *
B) ( (""7 E 2,
$ :
" & +,,D
6. SENTENCE RANKING
# # ( # &
Q 0 ; Q # '
* "
R
¾ S ½ S
¿
'G* ' R D + G* ' R D + G* D DJ 2
! "
D
2,
2,
2,
9 # ! &# # # 3D/4 $ ! "
'2* ¾ , J S D J S
' * R
G
+, $ 0$)
7.
PHRASE EXTRACTION AND RANKING
7.1 Potential answer identification $ # % # 3DL4 ; 2, # # 1 ' )-. /* ! " #$ D,GL, & D,GL, D, <1 X ? 1 D, D,GL, !
O " 8 ; G % (
activation weight 1
0
Pi-w
Pi-1
pi
Pi+1
$ +,
Pi+w
phrase position
7.2 Phrase ranking 9 - ) # # < ?
; 5 : '(1)90- X(-0$*
; <1 X ? # <( (? '(-)0&( (* 2J/ ;
; 2 '(&0*
# 3DL4 ; QK7 CC P 7 ' < ?* <(? & AI9 & ! # ! #
8. EXAMPLE : +D )-./" % $ <9 0 ? # ' ; J*
2, D2MDM
+V G,, 9 # <9 0 ? $ Æ ; J + ! 9 ! 9
; L ; M ; / U ' * # #
; N # # 9 + : # ; M < 0 ; & ? $
&
=#
+
QK7 7 ( 7 CC ( ( 7 (
& 'D,,V* & '/LMV* (-)0& 'G/V* B5K-) 'G/V* &) '+JV* (-)0& 'GM2V* ( 9.- '+NLV* 79 - '+DMV* & 'MLV* & 'MJLV* B5K-) 'DDDV* ( 9.- '22V* &) '22V* ( 9.- 'GMGV* (-)0& 'GJLV* & 'DLNV* &) 'D,+V* &) 'JJLV* & 'GGGV* ( 9.- 'JLV* 79 - 'JLV*
( -
' ' . # # ' / ' ' , ' !-
) 0 / !-1 2
4 @ 1 # C @ D *
E E
( ( E " "# # " $ ( ( 3 ( .# E " 5 E F " . # # 3 # C 5 @ 5 + 4 # " E 9 E % * 3 ' % +! . , '&
' 4
! "#"$ %% %%
; ' < C @ ? ! > ; ' < C
>C !@; 3$ ++. ++. 1 ++. ++ >C !C@< 5 ++. ++. ++. ++. >C !' ! 5 ++. ++. ++. >C ! ;C $1 " 84 >C C ' '' + ++. D ++. ++. ++. >C!? <> 3 84 >C!C< > $ 3$ ++ >C?@? ? DD ++ >C@!> ' ++. ++. >C@?'@ 3$ " ++ " ++ >C@C?C? ++. ( ( >C@;@;? , ++ , H 5+ >C@ ?>; ++ >C@ >@< " # +.+
5 6 7 + 4 , 4 2 !- 1 3 ( (455 3 ' ' ,
<7 ( ( ( ?
<(-)0&? <( 9.-? $ '<(-)0&?<7 ( ( ( ?* '<( 9.-?<7 ( ( ( ?*
R R
,
9 , &
# <9 0 ? ! D2MDM JDLGL/ #
GMLGGL
JDLGL/ M+//D2 1 > $ '<( 9.-?<( (?* S
'<(-)0&?<( (?*
GM+// S GJJNG ! )
8 (-)0& ( 9. ; <9 0 ? ' * DNGJ # J, # & )7) D/ ' D2 S ,G S ,D*
9.
EVALUATION
9.1 Performance by question length <@C> ; ++. ++. ><<@<@@ . / ++. 2/+30 ++. >< ' ?@ ++. ++. >' ><' 5 ++. ++. ++. ++. >' @;C> .0.I DD ++ ++ >' C C@ + ++. D ++. ++. ++. >' < ;; 3$ ++. ++. 2 ++. 5 # ++. >' > @' 3$ 0 ++. . ++. >'? ? ? ++. ++. #&"##" %% %% >'?C@@ " # ++. >'?'?' 5 ++. ++. ++. >'?;;! ++. >'? < $ ++. ++.
; ' < C
1 6 7 + 4 , 7 # 4 2 3 / 8 ' , ' #
; ' < C " ? ! > ; ' < C
><< ;'; " # ++. ><?C 6 ++. ><'?! C ++. ><' !'C 6 ++. ' "&$ %% %% ><;'!CC ++. ++. ><; '' ++. # ++. >< >@C ++. ++. ++ ><> < 2 ++. ++. >'!C?@' ++. ++. 3 # ++. $@> ++ >'! !@C 3$ ++. 5 ++. ++. ++. >'?>>'> ++. 0 ++. ++. ++. >'@@? < ++. ++. >'C >C! # ++. 0 ++. # ++. 0 ++.
9 6 7
7 + 4 , 7 # 4 2 / * : ' , 5 ' /
3 ' ' / 3 # % 4 4 (4 )4 8&
9 . 9 Y , . 9
"
0
,/GJJ
D,2NJ +JLDV
DGGLD JNN+V
D2N
DLG N2,V
DL2 D,,MV
1
3 !- 1 #
/ ) " + 2 6 - 2 1% - 2 1% - 2 1%
; ' ' >' ;! >;@;; ;C;C! ><' ? ;CC;! >C<>> >< '; - 4 !C';= !@<>= ! '= !;C = ?!!C= !>@!= ) " C;''= C ?'= ?;@'= ?> = J> = ;C @= '!<= C> = ) " ;;;;= ;>;>= '!'!= '@'@= 8 <9 ?
# ; J # 'DSDE+SDEGSDE2SDEJ* 9 #
'
* 0 ! #
9 )-. / ' /* # +, 0
