%% If newer version like 2.25.xx are used, `normalize-color` needs to be applied.
%% This should become the default once LSR moves to 2.26.
#(define my-color
  (if (eqv? (second (ly:version)) 25)
      (normalize-color white)
      white))

maskUpperTip =
#(lambda (grob)
   (let ((mask (ly:round-filled-box '(-1 . 1.25) '(-1.14 . 0) 0)))
     (ly:stencil-stack
      (ly:system-start-delimiter::print grob)
      Y UP
      (ly:make-stencil
       (list 'color my-color (ly:stencil-expr mask))))))

maskLowerTip =
#(lambda (grob)
   (let ((mask (ly:round-filled-box '(-1 . 1.25) '(-0.0 . 1.14) 0)))
     (ly:stencil-stack
      (ly:system-start-delimiter::print grob)
      Y DOWN
      (ly:make-stencil
       (list 'color my-color (ly:stencil-expr mask))))))

\markup {
  \score {
    \new StaffGroup  <<
      \new StaffGroup \with {
        \override SystemStartBracket.extra-offset = #'(0.6 . 0) 
      } <<
        \new Staff { c'^\markup \rotate #-90 ":-(" }
        \new Staff { d' }
      >>
      \new Staff { e' }
    >>
    \layout {}
  }
  "    "
  \score {
    \new StaffGroup  <<
      \new StaffGroup \with {
        \override SystemStartBracket.stencil = \maskUpperTip
        \override SystemStartBracket.extra-offset = #'(0.6 . 0)
      } <<
        \new Staff { c'^\markup \rotate #-90 ":-)" }
        \new Staff { d' }
      >>
      \new Staff { e' }
    >>
    \layout {}
  }
  "    "
  \score {
    \new StaffGroup  <<
      \new Staff { c'^\markup \rotate #-90 ":-)" }
      \new StaffGroup \with {
        \override SystemStartBracket.stencil = \maskLowerTip
        \override SystemStartBracket.extra-offset = #'(0.6 . 0)
      } <<
        \new Staff { d' }
        \new Staff { e' }
      >>
    >>
    \layout {}
  }
}